sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ AIPlatform/ EvaluationRun.ts

import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  hasAlchemyLabelMap,
  normalizeLocation,
  parseResourceName,
  toPhysicalRfc1035,
  userLabels,
} from "./helpers.ts";
import { waitForOperation } from "./operations.ts";

const MAX_NAME_LENGTH = 63;

export type EvaluationRunProps = {
  /**
   * Evaluation run id (the `{evaluation_run}` segment). Assigned by
   * Vertex on create. Provide to target an existing run.
   */
  evaluationRunId?: string;
  /**
   * Region. Immutable — changing it replaces the run.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * Display name. Defaults to a generated id when omitted. Immutable.
   */
  displayName?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * Arbitrary caller metadata.
   */
  metadata?: unknown;
  /**
   * Data source (an Evaluation Set name or a BigQuery request set).
   * Immutable.
   */
  dataSource: aiplatform.GoogleCloudAiplatformV1EvaluationRunDataSource;
  /**
   * Candidate-to-inference config map. Immutable.
   */
  inferenceConfigs?: Record<string, aiplatform.GoogleCloudAiplatformV1EvaluationRunInferenceConfig>;
  /**
   * Metrics, rubric, and output configuration. Immutable.
   */
  evaluationConfig?: aiplatform.GoogleCloudAiplatformV1EvaluationRunEvaluationConfig;
};

export type EvaluationRun = Resource<
  "GCP.AIPlatform.EvaluationRun",
  EvaluationRunProps,
  {
    /** Full resource name. */
    name: string;
    /** Evaluation run id (last path segment). */
    evaluationRunId: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** Display name. */
    displayName: string | undefined;
    /** Run state. */
    state: aiplatform.GoogleCloudAiplatformV1EvaluationRunStateEnum | (string & {}) | undefined;
    /** Data source. */
    dataSource: aiplatform.GoogleCloudAiplatformV1EvaluationRunDataSource | undefined;
    /** Inference configs. */
    inferenceConfigs: aiplatform.GoogleCloudAiplatformV1EvaluationRunInferenceConfigMap | undefined;
    /** Evaluation config. */
    evaluationConfig: aiplatform.GoogleCloudAiplatformV1EvaluationRunEvaluationConfig | undefined;
    /** Snapshot evaluation set. */
    evaluationSetSnapshot: string | undefined;
    /** Results when the run succeeded. */
    evaluationResults: aiplatform.GoogleCloudAiplatformV1EvaluationResults | undefined;
    /** Caller metadata. */
    metadata: unknown;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** Error when FAILED or CANCELLED. */
    error: aiplatform.GoogleRpcStatus | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 completion timestamp. */
    completionTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Vertex AI Evaluation Run — one execution of metrics over a set of
 * evaluation items or a BigQuery request set.
 *
 * Vertex assigns the resource id. Creating a run starts evaluation. There
 * is no update API: changing location, display name, data source, or
 * config replaces the run.
 *
 * ### Creating an Evaluation Run
 * **Example:** Run metrics over an evaluation set
 * ```typescript
 * const run = yield* GCP.AIPlatform.EvaluationRun("Quality", {
 *   dataSource: { evaluationSet: evaluationSet.name },
 *   evaluationConfig: {
 *     metrics: [
 *       {
 *         metric: "instruction_following_v1",
 *         predefinedMetricSpec: { metricSpecName: "instruction_following_v1" },
 *       },
 *     ],
 *   },
 *   labels: { env: "test" },
 * });
 * ```
 *
 * @resource
 * @category AIPlatform
 */
export const EvaluationRun = Resource<EvaluationRun>("GCP.AIPlatform.EvaluationRun");

export class EvaluationRunNotResolved extends Data.TaggedError(
  "GCP.AIPlatform.EvaluationRunNotResolved",
)<{
  resourceName: string;
}> {}

export class EvaluationRunStillExists extends Data.TaggedError(
  "GCP.AIPlatform.EvaluationRunStillExists",
)<{
  resourceName: string;
}> {}

const resourceName = (project: string, location: string, runId: string) =>
  `projects/${project}/locations/${location}/evaluationRuns/${runId}`;

const toAttrs = (run: aiplatform.GoogleCloudAiplatformV1EvaluationRun, project: string) => {
  const name = run.name ?? "";
  const parsed = parseResourceName(name, "evaluationRuns");
  return {
    name,
    evaluationRunId: parsed.id,
    project: parsed.project || project,
    location: parsed.location,
    displayName: run.displayName,
    state: run.state,
    dataSource: run.dataSource,
    inferenceConfigs: run.inferenceConfigs,
    evaluationConfig: run.evaluationConfig,
    evaluationSetSnapshot: run.evaluationSetSnapshot,
    evaluationResults: run.evaluationResults,
    metadata: run.metadata,
    labels: userLabels(run.labels),
    error: run.error,
    createTime: run.createTime,
    completionTime: run.completionTime,
  };
};

const getByName = (name: string) =>
  aiplatform
    .getProjectsLocationsEvaluationRuns({ name })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const listOwned = (project: string, location = "-") =>
  aiplatform.listProjectsLocationsEvaluationRuns
    .pages({
      parent: `projects/${project}/locations/${location}`,
      pageSize: 100,
    })
    .pipe(
      Stream.flatMap((page) => Stream.fromIterable(page.evaluationRuns ?? [])),
      Stream.filter((run) => hasAlchemyLabelMap(run.labels)),
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
      Effect.catchTag("NotFound", () => Effect.succeed([])),
    );

const findOwned = (id: string, project: string, location?: string) =>
  Effect.gen(function* () {
    const runs = yield* listOwned(project, location);
    for (const run of runs) {
      if (yield* hasAlchemyLabels(id, tagRecord(run.labels))) {
        return run;
      }
    }
    return undefined;
  });

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((run) =>
      run === undefined
        ? Effect.void
        : Effect.fail(new EvaluationRunStillExists({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.EvaluationRunStillExists",
      times: 10,
      schedule: Schedule.spaced("2 seconds"),
    }),
  );

export const EvaluationRunProvider = () =>
  Provider.succeed(EvaluationRun, {
    stables: ["name", "evaluationRunId", "project", "location", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      const env = yield* GcpEnvironment.current;
      if (!isResolved(news)) return undefined;
      const previousId = olds?.evaluationRunId ?? output?.evaluationRunId;
      const nextId = news.evaluationRunId ?? previousId;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
      const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
      const previousDisplay = olds?.displayName ?? output?.displayName ?? "";
      const nextDisplay = news.displayName ?? previousDisplay;
      const replace =
        (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
        previousLocation !== nextLocation ||
        (news.displayName !== undefined && nextDisplay !== previousDisplay);
      if (!replace) return undefined;
      return { action: "replace" as const, deleteFirst: false };
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const runId = olds?.evaluationRunId ?? output?.evaluationRunId;
      const name = output?.name ?? (runId ? resourceName(env.project, location, runId) : undefined);
      const existing = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const runs = yield* listOwned(env.project);
        return runs.map((run) => toAttrs(run, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const runId = news.evaluationRunId ?? output?.evaluationRunId;
      const name = output?.name ?? (runId ? resourceName(env.project, location, runId) : undefined);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };
      const displayName =
        news.displayName ?? (yield* toPhysicalRfc1035(id, undefined, undefined, MAX_NAME_LENGTH));

      let current = name ? yield* getByName(name) : yield* findOwned(id, env.project, location);

      if (current === undefined) {
        current = yield* aiplatform
          .createProjectsLocationsEvaluationRuns({
            parent: `projects/${env.project}/locations/${location}`,
            body: {
              displayName,
              dataSource: news.dataSource,
              inferenceConfigs: news.inferenceConfigs,
              evaluationConfig: news.evaluationConfig,
              metadata: news.metadata,
              labels: desiredLabels,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => findOwned(id, env.project, location)));
      }

      if (current === undefined) {
        return yield* new EvaluationRunNotResolved({
          resourceName: name ?? displayName,
        });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      // A running evaluation blocks the delete with BadRequest "There are
      // other operations running on the EvaluationRun" until it finishes
      // (cancel is unimplemented: HTTP 501).
      const operation = yield* aiplatform
        .deleteProjectsLocationsEvaluationRuns({ name: output.name })
        .pipe(
          Effect.catchTag("NotFound", () => Effect.undefined),
          Effect.retry({
            while: (error) => error._tag === "Conflict" || error._tag === "BadRequest",
            times: 24,
            schedule: Schedule.spaced("5 seconds"),
          }),
        );
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(output.name);
    }),
  });