sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ AIPlatform/ OnlineEvaluator.ts

import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import { listLocations } from "./names.ts";
import { resourceNameFromOperation, waitForOperation } from "./operations.ts";
import {
  AiPlatformNotResolved,
  AiPlatformStillExists,
  collectPages,
  encodeDisplayName,
  hasDisplayNameOwnership,
  jsonEqual,
  locationParent,
  normalizeLocation,
  ownedById,
  parseDisplayName,
  parseResourceName,
} from "./shared.ts";

const COLLECTION = "onlineEvaluators";

export type OnlineEvaluatorRandomSampling = {
  /** Percentage of traces to sample (`1`–`100`). */
  percentage?: number;
};

export type OnlineEvaluatorConfig = {
  /** Random sampling method. */
  randomSampling?: OnlineEvaluatorRandomSampling;
  /** Max evaluations per 10-minute run (`"0"` is unbounded). */
  maxEvaluatedSamplesPerRun?: string;
};

export type MetricSource = {
  /** Registered metric resource name. */
  metricResourceName?: string;
};

export type CloudObservability = {
  /** OpenTelemetry semantic convention version (e.g. `"1.39.0"`). */
  openTelemetry?: { semconvVersion?: string };
  /** Log view used to query logs. */
  logView?: string;
  /** Trace view used to query traces. */
  traceView?: string;
};

export type OnlineEvaluatorProps = {
  /**
   * Vertex AI location. Immutable — changing it replaces the evaluator.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Human-readable name (max 63 characters). OnlineEvaluator has no
   * labels field, so Alchemy ownership is stored in a compact `[alc …]`
   * displayName prefix for `list` / nuke.
   */
  displayName?: string;
  /**
   * Agent resource the evaluator scores. Immutable.
   */
  agentResource: string;
  /**
   * Metric sources (max 25). At least one is required.
   */
  metricSources: MetricSource[];
  /**
   * Sampling configuration. Defaults to 10% random sampling.
   */
  config?: OnlineEvaluatorConfig;
  /**
   * Cloud Observability (Trace / Logging) data source.
   */
  cloudObservability?: CloudObservability;
};

export type OnlineEvaluator = Resource<
  "GCP.AIPlatform.OnlineEvaluator",
  OnlineEvaluatorProps,
  {
    /** Full resource name. */
    name: string;
    /** Evaluator id (last path segment). */
    onlineEvaluatorId: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** User display name with the Alchemy ownership prefix stripped. */
    displayName: string | undefined;
    /** Agent resource being evaluated. */
    agentResource: string | undefined;
    /** Server-reported state (`ACTIVE`, `SUSPENDED`, `FAILED`, …). */
    state: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Vertex AI Online Evaluator — periodically scores agent traces against
 * registered metrics.
 *
 * OnlineEvaluator has no labels, so Alchemy stamps ownership into the
 * display name. Changing `location` or `agentResource` replaces the
 * evaluator. Display name, config, and metric sources update in place.
 *
 * ### Creating an Online Evaluator
 * **Example:** Sample 10% of traces
 * ```typescript
 * const evaluator = yield* GCP.AIPlatform.OnlineEvaluator("Quality", {
 *   agentResource: engine.name,
 *   metricSources: [{ metricResourceName: metric.name }],
 *   config: { randomSampling: { percentage: 10 } },
 * });
 * ```
 *
 * @resource
 * @category AIPlatform
 */
export const OnlineEvaluator = Resource<OnlineEvaluator>("GCP.AIPlatform.OnlineEvaluator");

export class OnlineEvaluatorNotResolved extends Data.TaggedError(
  "GCP.AIPlatform.OnlineEvaluatorNotResolved",
)<{
  resourceName: string;
}> {}

const defaultConfig = (
  config: OnlineEvaluatorConfig | undefined,
): aiplatform.GoogleCloudAiplatformV1OnlineEvaluatorConfig => ({
  randomSampling: {
    percentage: config?.randomSampling?.percentage ?? 10,
  },
  maxEvaluatedSamplesPerRun: config?.maxEvaluatedSamplesPerRun,
});

const toAttrs = (evaluator: aiplatform.GoogleCloudAiplatformV1OnlineEvaluator, project: string) => {
  const name = evaluator.name ?? "";
  const parsed = parseResourceName(name, COLLECTION);
  const display = parseDisplayName(evaluator.displayName);
  return {
    name,
    onlineEvaluatorId: parsed.id,
    project: parsed.project || project,
    location: parsed.location,
    displayName: display.displayName,
    agentResource: evaluator.agentResource,
    state: evaluator.state,
    createTime: evaluator.createTime,
    updateTime: evaluator.updateTime,
  };
};

const getByName = (name: string) =>
  aiplatform
    .getProjectsLocationsOnlineEvaluators({ name })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const waitUntilExists = (name: string) =>
  getByName(name).pipe(
    Effect.filterOrFail(
      (evaluator): evaluator is aiplatform.GoogleCloudAiplatformV1OnlineEvaluator =>
        evaluator !== undefined,
      () => new AiPlatformNotResolved({ resourceName: name }),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.NotResolved",
      times: 8,
      schedule: Schedule.spaced("2 seconds"),
    }),
  );

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.filterOrFail(
      (evaluator) => evaluator === undefined,
      () => new AiPlatformStillExists({ resourceName: name }),
    ),
    Effect.asVoid,
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.StillExists",
      times: 10,
      schedule: Schedule.spaced("2 seconds"),
    }),
  );

const findOwned = (parent: string, id: string) =>
  Effect.gen(function* () {
    const pages = yield* collectPages(
      aiplatform.listProjectsLocationsOnlineEvaluators.pages({
        parent,
        pageSize: 100,
      }),
    ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([])));
    const evaluators = pages.flatMap((page) => page.onlineEvaluators ?? []);
    for (const evaluator of evaluators) {
      const parsed = parseDisplayName(evaluator.displayName);
      if (yield* ownedById(id, parsed.labels)) {
        return evaluator;
      }
    }
    return undefined;
  });

export const OnlineEvaluatorProvider = () =>
  Provider.succeed(OnlineEvaluator, {
    stables: ["name", "onlineEvaluatorId", "project", "location", "agentResource", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      const env = yield* GcpEnvironment.current;
      if (!isResolved(news)) return undefined;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
      const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
      const agentChanged =
        news.agentResource !== (olds?.agentResource ?? output?.agentResource ?? news.agentResource);
      const replace = previousLocation !== nextLocation || (olds !== undefined && agentChanged);
      if (!replace) return undefined;
      return {
        action: "replace" as const,
        deleteFirst: previousLocation === nextLocation,
      };
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const parent = locationParent(env.project, location);
      const existing =
        output?.name !== undefined ? yield* getByName(output.name) : yield* findOwned(parent, id);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      const parsed = parseDisplayName(existing.displayName);
      return (yield* hasAlchemyLabels(id, parsed.labels)) || (yield* ownedById(id, parsed.labels))
        ? attrs
        : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const pages = (yield* Effect.forEach(listLocations(env.region), (location) =>
          collectPages(
            aiplatform.listProjectsLocationsOnlineEvaluators.pages({
              parent: locationParent(env.project, location),
              pageSize: 100,
            }),
          ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))),
        )).flat();
        return pages.flatMap((page) =>
          (page.onlineEvaluators ?? [])
            .filter((evaluator) => hasDisplayNameOwnership(evaluator.displayName))
            .map((evaluator) => toAttrs(evaluator, env.project)),
        );
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const parent = locationParent(env.project, location);
      const internal = yield* createInternalLabels(id);
      const displayName = encodeDisplayName(internal, news.displayName);
      const config = defaultConfig(news.config);

      let current =
        output?.name !== undefined ? yield* getByName(output.name) : yield* findOwned(parent, id);

      if (current === undefined) {
        const created = yield* aiplatform.createProjectsLocationsOnlineEvaluators({
          parent,
          body: {
            displayName,
            agentResource: news.agentResource,
            metricSources: news.metricSources,
            config,
            cloudObservability: news.cloudObservability,
          },
        });
        yield* waitForOperation(created, { alreadyExistsOk: true });
        const createdName = resourceNameFromOperation(created);
        current =
          createdName !== undefined
            ? yield* waitUntilExists(createdName)
            : yield* findOwned(parent, id);
      }

      if (current === undefined || current.name === undefined) {
        return yield* new OnlineEvaluatorNotResolved({
          resourceName: output?.name ?? parent,
        });
      }

      const observedName = current.name;
      const observedDisplay = parseDisplayName(current.displayName);
      const displayChanged = (observedDisplay.displayName ?? "") !== (news.displayName ?? "");
      const configChanged = !jsonEqual(current.config, config);
      const metricsChanged = !jsonEqual(current.metricSources, news.metricSources);

      if (displayChanged || configChanged || metricsChanged) {
        const updateMask = [
          displayChanged ? "display_name" : undefined,
          configChanged ? "config" : undefined,
          metricsChanged ? "metric_sources" : undefined,
        ].filter((field): field is string => field !== undefined);
        const patched = yield* aiplatform.patchProjectsLocationsOnlineEvaluators({
          name: observedName,
          updateMask: updateMask.join(","),
          body: {
            name: observedName,
            displayName,
            config,
            metricSources: news.metricSources,
          },
        });
        yield* waitForOperation(patched);
        current = yield* getByName(observedName);
      }

      if (current === undefined) {
        return yield* new OnlineEvaluatorNotResolved({ resourceName: observedName });
      }
      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      const operation = yield* aiplatform
        .deleteProjectsLocationsOnlineEvaluators({ name: output.name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(output.name);
    }),
  });