sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ DataLabeling/ EvaluationJob.ts

import * as datalabeling from "@distilled.cloud/gcp/datalabeling_v1beta1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  encodeOwnership,
  findOwned,
  hasOwnershipMarker,
  ignoreGone,
  listEvaluationJobs,
  MAX_EVALUATION_DESCRIPTION_LENGTH,
  noRetryLayer,
  ownedByAlchemy,
  parseOwnership,
  parseResourceName,
  projectParent,
  replaceOnIdentity,
  retryDelete,
  retryTransient,
  sameJson,
  sameText,
  updateMaskOf,
  waitUntilGone,
} from "./internal.ts";

export type EvaluationJobConfig = datalabeling.GoogleCloudDatalabelingV1beta1EvaluationJobConfig;

export type EvaluationJobProps = {
  /**
   * Evaluation job id (the last segment of
   * `projects/{project}/evaluationJobs/{evaluation_job}`).
   * Server-assigned on create. Immutable — changing it replaces the
   * job.
   */
  evaluationJobId?: string;
  /**
   * Description of the job. Maximum 25,000 characters. Evaluation jobs
   * have no labels field, so Alchemy stamps ownership into this field.
   * Immutable — changing it replaces the job.
   */
  description?: string;
  /**
   * Annotation spec set resource name describing the labels the model
   * outputs. Immutable — changing it replaces the job.
   */
  annotationSpecSet: string;
  /**
   * AI Platform Prediction model version sampled for evaluation, as
   * `projects/{project}/models/{model}/versions/{version}`. Immutable —
   * changing it replaces the job.
   */
  modelVersion: string;
  /**
   * Run interval, in crontab or English-like format. Must be at least
   * one day. Immutable — changing it replaces the job.
   */
  schedule: string;
  /**
   * When true, Data Labeling assigns human labelers for missing ground
   * truth. When false, ground truth is read from the job's BigQuery
   * table.
   * @default false
   */
  labelMissingGroundTruth?: boolean;
  /**
   * Configuration for sampling, import keys, human annotation, and
   * metrics. Only `humanAnnotationConfig.instruction`, `exampleCount`,
   * and `exampleSamplePercentage` update in place; any other change
   * replaces the job.
   */
  evaluationJobConfig: EvaluationJobConfig;
  /**
   * When true, pause the job after ensure. Pausing a paused job is a
   * no-op; resuming a running or scheduled job is a no-op.
   * @default false
   */
  paused?: boolean;
};

export type EvaluationJob = Resource<
  "GCP.DataLabeling.EvaluationJob",
  EvaluationJobProps,
  {
    /** Full resource name `projects/{project}/evaluationJobs/{job}`. */
    name: string;
    /** Evaluation job id (last path segment). */
    evaluationJobId: string;
    /** Project id. */
    project: string;
    /** User description with the Alchemy ownership prefix stripped. */
    description: string | undefined;
    /** Annotation spec set resource name. */
    annotationSpecSet: string | undefined;
    /** Model version sampled for evaluation. */
    modelVersion: string | undefined;
    /** Run interval. */
    schedule: string | undefined;
    /** Whether the service provides ground-truth labels. */
    labelMissingGroundTruth: boolean;
    /** Job configuration. */
    evaluationJobConfig: EvaluationJobConfig | undefined;
    /** Server-reported job state. */
    state: string | undefined;
    /** Whether the job is paused. */
    paused: boolean;
    /** Failed attempts, if any. */
    attempts: datalabeling.GoogleCloudDatalabelingV1beta1AttemptList | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Data Labeling evaluation job that periodically samples predictions
 * from an AI Platform model version and scores them.
 *
 * Patch can only update `evaluationJobConfig.humanAnnotationConfig.instruction`,
 * `evaluationJobConfig.exampleCount`, and
 * `evaluationJobConfig.exampleSamplePercentage`. Pause and resume are
 * separate RPCs. Every other field is identity — changing it replaces
 * the job. There is no labels API, so Alchemy stamps ownership into
 * `description`.
 *
 * ### Creating an Evaluation Job
 * **Example:** Daily image classification eval
 * ```typescript
 * const evaluationJobConfig = {
 *   exampleCount: 100,
 *   exampleSamplePercentage: 0.1,
 *   evaluationConfig: {},
 *   inputConfig: {
 *     dataType: "IMAGE",
 *     annotationType: "IMAGE_CLASSIFICATION_ANNOTATION",
 *     classificationMetadata: { isMultiLabel: false },
 *     bigquerySource: {
 *       inputUri: "bq://my-project.eval.predictions",
 *     },
 *   },
 *   bigqueryImportKeys: {
 *     data_json_key: "data",
 *     label_json_key: "label",
 *     label_score_json_key: "score",
 *   },
 *   imageClassificationConfig: {
 *     annotationSpecSet: specs.name,
 *   },
 * };
 * const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
 *   annotationSpecSet: specs.name,
 *   modelVersion:
 *     "projects/my-project/models/classifier/versions/v1",
 *   schedule: "every 24 hours",
 *   evaluationJobConfig,
 * });
 * ```
 *
 * ### Pausing an Evaluation Job
 * **Example:** Pause sampling
 * ```typescript
 * // Same logical id, changed props: the engine updates it in place.
 * const job = yield* GCP.DataLabeling.EvaluationJob("DailyEval", {
 *   annotationSpecSet: specs.name,
 *   modelVersion:
 *     "projects/my-project/models/classifier/versions/v1",
 *   schedule: "every 24 hours",
 *   evaluationJobConfig,
 *   paused: true,
 * });
 * ```
 *
 * @resource
 * @category DataLabeling
 */
export const EvaluationJob = Resource<EvaluationJob>("GCP.DataLabeling.EvaluationJob");

export class EvaluationJobNotResolved extends Data.TaggedError(
  "GCP.DataLabeling.EvaluationJobNotResolved",
)<{
  resourceName: string;
}> {}

const resourceName = (project: string, evaluationJobId: string) =>
  `${projectParent(project)}/evaluationJobs/${evaluationJobId}`;

const isPaused = (state: string | undefined) => state === "PAUSED";

const configIdentity = (config: EvaluationJobConfig | undefined) => {
  if (config === undefined) return undefined;
  const {
    exampleCount: _exampleCount,
    exampleSamplePercentage: _exampleSamplePercentage,
    humanAnnotationConfig,
    ...rest
  } = config;
  const { instruction: _instruction, ...humanRest } = humanAnnotationConfig ?? {};
  return {
    ...rest,
    humanAnnotationConfig: Object.keys(humanRest).length > 0 ? humanRest : undefined,
  };
};

const toAttrs = (
  job: datalabeling.GoogleCloudDatalabelingV1beta1EvaluationJob,
  project: string,
) => {
  const name = job.name ?? "";
  const parsed = parseResourceName(name, "evaluationJobs");
  return {
    name,
    evaluationJobId: parsed.id,
    project: parsed.project || project,
    description: parseOwnership(job.description).text,
    annotationSpecSet: job.annotationSpecSet,
    modelVersion: job.modelVersion,
    schedule: job.schedule,
    labelMissingGroundTruth: job.labelMissingGroundTruth === true,
    evaluationJobConfig: job.evaluationJobConfig,
    state: job.state,
    paused: isPaused(job.state),
    attempts: job.attempts,
    createTime: job.createTime,
  };
};

const getByName = (name: string) =>
  name.length === 0
    ? Effect.undefined
    : datalabeling.getProjectsEvaluationJobs({ name }).pipe(
        Effect.provide(noRetryLayer),
        Effect.catchTags({
          NotFound: () => Effect.undefined,
          BadGateway: () => Effect.undefined,
        }),
      );

const findByOwnership = (id: string, project: string) =>
  Effect.gen(function* () {
    const rows = yield* listEvaluationJobs(projectParent(project));
    return yield* findOwned(id, rows, (row) => row.description);
  });

export const EvaluationJobProvider = () =>
  Provider.succeed(EvaluationJob, {
    stables: ["name", "evaluationJobId", "project", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const extra =
        (output?.annotationSpecSet !== undefined &&
          !sameText(news.annotationSpecSet, output.annotationSpecSet)) ||
        (output?.modelVersion !== undefined && !sameText(news.modelVersion, output.modelVersion)) ||
        (output?.schedule !== undefined && !sameText(news.schedule, output.schedule)) ||
        (output !== undefined &&
          (news.labelMissingGroundTruth === true) !== output.labelMissingGroundTruth) ||
        (olds !== undefined && !sameText(news.description, output?.description)) ||
        (output !== undefined &&
          !sameJson(
            configIdentity(news.evaluationJobConfig),
            configIdentity(output.evaluationJobConfig),
          ));
      return replaceOnIdentity({
        previousId: olds?.evaluationJobId ?? output?.evaluationJobId,
        nextId: news.evaluationJobId,
        extra,
      });
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const evaluationJobId =
        olds?.evaluationJobId ??
        output?.evaluationJobId ??
        (output?.name ? parseResourceName(output.name, "evaluationJobs").id : "");
      const name =
        output?.name ??
        (evaluationJobId.length > 0 ? resourceName(env.project, evaluationJobId) : "");
      const existing = (yield* getByName(name)) ?? (yield* findByOwnership(id, env.project));
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* ownedByAlchemy(id, existing.description)) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const rows = yield* listEvaluationJobs(projectParent(env.project));
        return rows
          .filter((row) => hasOwnershipMarker(row.description))
          .map((row) => toAttrs(row, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const evaluationJobId = news.evaluationJobId ?? output?.evaluationJobId;
      const name =
        output?.name ??
        (evaluationJobId !== undefined ? resourceName(env.project, evaluationJobId) : "");
      const ownership = yield* createInternalLabels(id);
      const description = encodeOwnership(
        ownership,
        news.description,
        MAX_EVALUATION_DESCRIPTION_LENGTH,
      );
      const labelMissingGroundTruth = news.labelMissingGroundTruth === true;
      const desiredPaused = news.paused === true;

      let current = (yield* getByName(name)) ?? (yield* findByOwnership(id, env.project));

      if (current === undefined) {
        const created = yield* retryTransient(
          datalabeling.createProjectsEvaluationJobs({
            parent: projectParent(env.project),
            body: {
              job: {
                description,
                annotationSpecSet: news.annotationSpecSet,
                modelVersion: news.modelVersion,
                schedule: news.schedule,
                labelMissingGroundTruth,
                evaluationJobConfig: news.evaluationJobConfig,
              },
            },
          }),
        ).pipe(Effect.catchTag("Conflict", () => findByOwnership(id, env.project)));
        current = created ?? undefined;
      }

      if (current === undefined) {
        return yield* new EvaluationJobNotResolved({
          resourceName: name || projectParent(env.project),
        });
      }

      const currentName = current.name ?? name;
      const observedConfig = current.evaluationJobConfig;
      const desiredConfig = news.evaluationJobConfig;
      const instructionChanged = !sameText(
        observedConfig?.humanAnnotationConfig?.instruction,
        desiredConfig.humanAnnotationConfig?.instruction,
      );
      const exampleCountChanged =
        (observedConfig?.exampleCount ?? undefined) !== (desiredConfig.exampleCount ?? undefined);
      const sampleChanged =
        (observedConfig?.exampleSamplePercentage ?? undefined) !==
        (desiredConfig.exampleSamplePercentage ?? undefined);

      if (instructionChanged || exampleCountChanged || sampleChanged) {
        current = yield* retryTransient(
          datalabeling.patchProjectsEvaluationJobs({
            name: currentName,
            updateMask: updateMaskOf(
              instructionChanged
                ? "evaluationJobConfig.humanAnnotationConfig.instruction"
                : undefined,
              exampleCountChanged ? "evaluationJobConfig.exampleCount" : undefined,
              sampleChanged ? "evaluationJobConfig.exampleSamplePercentage" : undefined,
            ),
            body: {
              evaluationJobConfig: {
                humanAnnotationConfig: desiredConfig.humanAnnotationConfig,
                exampleCount: desiredConfig.exampleCount,
                exampleSamplePercentage: desiredConfig.exampleSamplePercentage,
              },
            },
          }),
        );
      }

      if (desiredPaused && !isPaused(current.state)) {
        yield* retryTransient(
          datalabeling.pauseProjectsEvaluationJobs({
            name: currentName,
            body: {},
          }),
        );
        current = (yield* getByName(currentName)) ?? current;
      } else if (!desiredPaused && isPaused(current.state)) {
        yield* retryTransient(
          datalabeling.resumeProjectsEvaluationJobs({
            name: currentName,
            body: {},
          }),
        );
        current = (yield* getByName(currentName)) ?? current;
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      if (!output.name) return;
      yield* datalabeling
        .deleteProjectsEvaluationJobs({ name: output.name })
        .pipe(retryDelete, ignoreGone);
      yield* waitUntilGone(getByName(output.name));
    }),
  });