sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ AIPlatform/ DatasetsDatasetVersion.ts

import { createHash } from "node:crypto";
import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, hasAlchemyLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  hasAlchemyLabelKeys,
  parseOwnership,
  parseResourceName,
  resourceNameFromOperation,
  waitForOperation,
} from "./internal.ts";
import { listLocations } from "./names.ts";

export type DatasetsDatasetVersionProps = {
  /**
   * Parent Dataset resource name
   * `projects/{project}/locations/{location}/datasets/{dataset}`.
   * Immutable — changing it replaces the version.
   */
  dataset: string;
  /**
   * User-facing display name (max 128 characters). Alchemy stamps
   * ownership into this field (versions have no labels) so `list` /
   * nuke can find the version.
   */
  displayName?: string;
};

export type DatasetsDatasetVersion = Resource<
  "GCP.AIPlatform.DatasetsDatasetVersion",
  DatasetsDatasetVersionProps,
  {
    /** Full resource name `.../datasets/{dataset}/datasetVersions/{version}`. */
    name: string;
    /** Dataset version id (last path segment). */
    datasetVersionId: string;
    /** Parent dataset resource name. */
    dataset: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** User display name with the Alchemy ownership prefix stripped. */
    displayName: string | undefined;
    /** Schema-specific metadata. */
    metadata: unknown;
    /** Associated BigQuery dataset name, if any. */
    bigQueryDatasetName: string | undefined;
    /** Public base model reference, if any. */
    modelReference: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Vertex AI DatasetVersion — a snapshot of a Dataset.
 *
 * Versions have no labels field, so Alchemy stamps ownership into
 * `displayName` for `list` / nuke. The parent dataset is immutable;
 * display name updates in place.
 *
 * ### Creating a Dataset Version
 * **Example:** Snapshot a dataset
 * ```typescript
 * const dataset = yield* GCP.AIPlatform.Dataset("Samples", {
 *   metadataSchemaUri:
 *     "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml",
 * });
 * const version = yield* GCP.AIPlatform.DatasetsDatasetVersion("V1", {
 *   dataset: dataset.name,
 *   displayName: "v1",
 * });
 * ```
 *
 * ### Updating a Dataset Version
 * **Example:** Rename the snapshot
 * ```typescript
 * // Same logical id, changed props: the engine updates it in place.
 * const version = yield* GCP.AIPlatform.DatasetsDatasetVersion("V1", {
 *   dataset: dataset.name,
 *   displayName: "v1-final",
 * });
 * ```
 *
 * @resource
 * @category AIPlatform
 */
export const DatasetsDatasetVersion = Resource<DatasetsDatasetVersion>(
  "GCP.AIPlatform.DatasetsDatasetVersion",
);

export class DatasetsDatasetVersionNotResolved extends Data.TaggedError(
  "GCP.AIPlatform.DatasetsDatasetVersionNotResolved",
)<{
  resourceName: string;
}> {}

export class DatasetsDatasetVersionStillExists extends Data.TaggedError(
  "GCP.AIPlatform.DatasetsDatasetVersionStillExists",
)<{
  resourceName: string;
}> {}

const parentDataset = (name: string) => {
  const marker = "/datasetVersions/";
  const index = name.indexOf(marker);
  return index >= 0 ? name.slice(0, index) : name;
};

const parseVersionName = (name: string) => {
  const parsed = parseResourceName(name, "datasetVersions");
  const datasets = parseResourceName(name, "datasets");
  return {
    ...parsed,
    datasetId: datasets.id,
    dataset: parentDataset(name),
  };
};

const getByName = (name: string) =>
  name.length === 0
    ? Effect.undefined
    : aiplatform
        .getDatasetsDatasetVersions({ name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

/**
 * Dataset versions carry ownership only in their display name, which
 * Vertex AI caps at 128 characters — too short for the full
 * `[alchemy stack=… stage=… id=…]` marker of long stack names. Stamp a
 * fixed-width hash of the ownership labels instead.
 */
const ownerTag = (id: string) =>
  createInternalLabels(id).pipe(
    Effect.flatMap((labels) =>
      Effect.sync(
        () =>
          `[alchemy #${createHash("sha256")
            .update(JSON.stringify(labels))
            .digest("hex")
            .slice(0, 16)}]`,
      ),
    ),
  );

const encodeDisplayName = (tag: string, displayName: string | undefined) =>
  displayName ? `${tag} ${displayName}` : tag;

const ownedByAlchemy = (id: string, displayName: string | undefined) =>
  Effect.gen(function* () {
    if (displayName?.startsWith(yield* ownerTag(id))) return true;
    return yield* hasAlchemyLabels(id, parseOwnership(displayName).labels);
  });

const toAttrs = (version: aiplatform.GoogleCloudAiplatformV1DatasetVersion, project: string) => {
  const name = version.name ?? "";
  const parsed = parseVersionName(name);
  const ownership = parseOwnership(version.displayName);
  return {
    name,
    datasetVersionId: parsed.id,
    dataset: parsed.dataset,
    project: parsed.project || project,
    location: parsed.location,
    displayName: ownership.text,
    metadata: version.metadata,
    bigQueryDatasetName: version.bigQueryDatasetName,
    modelReference: version.modelReference,
    createTime: version.createTime,
    updateTime: version.updateTime,
  };
};

const listVersions = (dataset: string) =>
  aiplatform.listProjectsLocationsDatasetsDatasetVersions
    .pages({ parent: dataset, pageSize: 100 })
    .pipe(
      Stream.flatMap((page) => Stream.fromIterable(page.datasetVersions ?? [])),
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
      Effect.catchTag("NotFound", () => Effect.succeed([])),
    );

const listDatasets = (project: string, region: string) =>
  Effect.forEach(listLocations(region), (location) =>
    listDatasetsAt(`projects/${project}/locations/${location}`),
  ).pipe(Effect.map((pages) => pages.flat()));

const listDatasetsAt = (parent: string) =>
  aiplatform.listProjectsLocationsDatasets
    .pages({
      parent,
      pageSize: 100,
    })
    .pipe(
      Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
      Stream.filter((dataset) => hasAlchemyLabelKeys(dataset.labels)),
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
      Effect.catchTag("NotFound", () => Effect.succeed([])),
    );

const findOwned = (id: string, dataset: string, hinted?: string) =>
  Effect.gen(function* () {
    if (hinted !== undefined && hinted.length > 0) {
      const existing = yield* getByName(hinted);
      if (existing !== undefined) return existing;
    }
    const versions = yield* listVersions(dataset);
    for (const version of versions) {
      if (yield* ownedByAlchemy(id, version.displayName)) return version;
    }
    return undefined as aiplatform.GoogleCloudAiplatformV1DatasetVersion | undefined;
  });

const waitUntilExists = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((version) =>
      version
        ? Effect.succeed(version)
        : Effect.fail(new DatasetsDatasetVersionNotResolved({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.DatasetsDatasetVersionNotResolved",
      times: 8,
      schedule: Schedule.spaced("1 second"),
    }),
  );

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((version) =>
      version === undefined
        ? Effect.void
        : Effect.fail(new DatasetsDatasetVersionStillExists({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.DatasetsDatasetVersionStillExists",
      times: 10,
      schedule: Schedule.spaced("2 seconds"),
    }),
  );

export const DatasetsDatasetVersionProvider = () =>
  Provider.succeed(DatasetsDatasetVersion, {
    stables: ["name", "datasetVersionId", "dataset", "project", "location", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;
      const previous = olds?.dataset ?? output?.dataset;
      if (previous !== undefined && news.dataset !== previous) {
        return { action: "replace" as const, deleteFirst: false };
      }
      return undefined;
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const dataset = olds?.dataset ?? output?.dataset;
      const existing =
        output?.name !== undefined
          ? yield* getByName(output.name)
          : dataset !== undefined
            ? yield* findOwned(id, dataset)
            : undefined;
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* ownedByAlchemy(id, existing.displayName)) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const datasets = yield* listDatasets(env.project, env.region);
        const versions: ReturnType<typeof toAttrs>[] = [];
        for (const dataset of datasets) {
          if (dataset.name === undefined) continue;
          const page = yield* listVersions(dataset.name);
          for (const version of page) {
            if (version.displayName?.startsWith("[alchemy ")) {
              versions.push(toAttrs(version, env.project));
            }
          }
        }
        return versions;
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const desiredDisplayName = encodeDisplayName(yield* ownerTag(id), news.displayName);

      let current = yield* findOwned(id, news.dataset, output?.name);

      if (current === undefined) {
        const created = yield* aiplatform
          .createDatasetsDatasetVersions({
            parent: news.dataset,
            body: { displayName: desiredDisplayName },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        if (created !== undefined) {
          const done = yield* waitForOperation(created);
          const createdName =
            resourceNameFromOperation(done) ?? (yield* findOwned(id, news.dataset))?.name;
          if (createdName !== undefined && createdName.length > 0) {
            current = yield* waitUntilExists(createdName);
          }
        }
        if (current === undefined) {
          current = yield* findOwned(id, news.dataset);
        }
      }

      if (current === undefined) {
        return yield* new DatasetsDatasetVersionNotResolved({
          resourceName: output?.name ?? `${news.dataset}/datasetVersions/-`,
        });
      }

      const name = current.name ?? "";
      if ((current.displayName ?? "") !== desiredDisplayName) {
        current = yield* aiplatform.patchProjectsLocationsDatasetsDatasetVersions({
          name,
          updateMask: "display_name",
          body: { name, displayName: desiredDisplayName },
        });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      const existing = yield* getByName(output.name);
      if (existing === undefined) return;
      const operation = yield* aiplatform
        .deleteDatasetsDatasetVersions({ name: output.name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(output.name);
    }),
  });