sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ AIPlatform/ Dataset.ts

import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  DEFAULT_TABULAR_METADATA_SCHEMA_URI,
  hasAlchemyLabelKeys,
  normalizeLocation,
  parentOf,
  parseResourceName,
  rfc1035,
  resourceNameFromOperation,
  userLabels,
  waitForOperation,
} from "./internal.ts";
import { listLocations } from "./names.ts";
import type { EncryptionSpec } from "./shared.ts";

export type DatasetProps = {
  /**
   * Dataset id (the `{dataset}` segment of
   * `projects/{project}/locations/{location}/datasets/{dataset}`). If
   * omitted, Vertex AI assigns an id. Immutable — changing it replaces
   * the dataset.
   */
  datasetId?: string;
  /**
   * Region (`us-central1`, `us-east1`, …). Immutable — changing it
   * replaces the dataset. `US-CENTRAL1` is accepted and normalized to
   * `us-central1`.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * User-facing display name (max 128 characters). Defaults to the
   * dataset id.
   */
  displayName?: string;
  /**
   * Human-readable description.
   */
  description?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * GCS URI of the dataset metadata schema YAML (OpenAPI 3.0.2 Schema
   * Object). Immutable — changing it replaces the dataset.
   * @default "gs://google-cloud-aiplatform/schema/dataset/metadata/tabular_1.0.0.yaml"
   */
  metadataSchemaUri?: string;
  /**
   * Additional schema-specific metadata. For an empty tabular dataset,
   * `{}` is sufficient.
   */
  metadata?: unknown;
  /**
   * Customer-managed encryption. Immutable — changing it replaces the
   * dataset.
   */
  encryptionSpec?: EncryptionSpec;
  /**
   * Public base model last used by a prompt dataset.
   */
  modelReference?: string;
  /**
   * Saved queries created with the dataset (max one on create). Required
   * for DatasetVersion — Vertex rejects versioning an empty dataset with
   * no SavedQuery (`Unsupported dataset version creation with empty set
   * of saved queries`).
   */
  savedQueries?: Array<{
    /** User-facing saved query name. */
    displayName?: string;
    /**
     * Problem type (`IMAGE_CLASSIFICATION_SINGLE_LABEL`,
     * `TEXT_CLASSIFICATION_SINGLE_LABEL`, …).
     */
    problemType?: string;
    /** Additional saved-query metadata. */
    metadata?: unknown;
  }>;
};

export type Dataset = Resource<
  "GCP.AIPlatform.Dataset",
  DatasetProps,
  {
    /** Full resource name `projects/{project}/locations/{location}/datasets/{dataset}`. */
    name: string;
    /** Dataset id (last path segment). */
    datasetId: string;
    /** Project id. */
    project: string;
    /** Location id (`us-central1`, …). */
    location: string;
    /** User-facing display name. */
    displayName: string | undefined;
    /** Description. */
    description: string | undefined;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** Metadata schema URI. */
    metadataSchemaUri: string | undefined;
    /** Schema-specific metadata. */
    metadata: unknown;
    /** Number of DataItems (non-structured datasets). */
    dataItemCount: string | undefined;
    /** MetadataStore artifact created with the dataset. */
    metadataArtifact: string | undefined;
    /** Public base model reference, if any. */
    modelReference: string | undefined;
    /** Customer-managed KMS key, if any. */
    kmsKeyName: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Vertex AI Dataset — a collection of DataItems and Annotations.
 *
 * Location, dataset id, metadata schema, and CMEK are immutable. Display
 * name, description, and labels update in place. Alchemy ownership labels
 * are merged into `labels` so `list` / `pnpm nuke:gcp` can find the
 * dataset.
 *
 * ### Creating a Dataset
 * **Example:** Generated id, empty tabular dataset
 * ```typescript
 * const dataset = yield* GCP.AIPlatform.Dataset("Samples", {
 *   displayName: "sample rows",
 *   labels: { env: "test" },
 * });
 * ```
 *
 * **Example:** Explicit id and image metadata schema
 * ```typescript
 * const dataset = yield* GCP.AIPlatform.Dataset("Images", {
 *   datasetId: "app-images",
 *   location: "us-central1",
 *   metadataSchemaUri:
 *     "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml",
 *   metadata: {},
 * });
 * ```
 *
 * ### Updating a Dataset
 * **Example:** Rename and relabel
 * ```typescript
 * // Same logical id, changed props: the engine updates it in place.
 * const dataset = yield* GCP.AIPlatform.Dataset("Samples", {
 *   displayName: "prod rows",
 *   labels: { env: "prod" },
 * });
 * ```
 *
 * @resource
 * @category AIPlatform
 */
export const Dataset = Resource<Dataset>("GCP.AIPlatform.Dataset");

export class DatasetNotResolved extends Data.TaggedError("GCP.AIPlatform.DatasetNotResolved")<{
  resourceName: string;
}> {}

export class DatasetStillExists extends Data.TaggedError("GCP.AIPlatform.DatasetStillExists")<{
  resourceName: string;
}> {}

const resourceName = (project: string, location: string, datasetId: string) =>
  `projects/${project}/locations/${location}/datasets/${datasetId}`;

const toId = (id: string, datasetId: string | undefined, existing?: string) =>
  Effect.gen(function* () {
    if (datasetId !== undefined) return rfc1035(datasetId);
    if (existing !== undefined) return existing;
    return rfc1035(
      yield* createPhysicalName({
        id,
        maxLength: 63,
        lowercase: true,
      }),
    );
  });

const schemaUriOf = (uri: string | undefined) =>
  uri && uri.length > 0 ? uri : DEFAULT_TABULAR_METADATA_SCHEMA_URI;

const toAttrs = (dataset: aiplatform.GoogleCloudAiplatformV1Dataset, project: string) => {
  const name = dataset.name ?? "";
  const parsed = parseResourceName(name, "datasets");
  return {
    name,
    datasetId: parsed.id,
    project: parsed.project || project,
    location: parsed.location,
    displayName: dataset.displayName,
    description: dataset.description,
    labels: userLabels(dataset.labels),
    metadataSchemaUri: dataset.metadataSchemaUri,
    metadata: dataset.metadata,
    dataItemCount: dataset.dataItemCount,
    metadataArtifact: dataset.metadataArtifact,
    modelReference: dataset.modelReference,
    kmsKeyName: dataset.encryptionSpec?.kmsKeyName,
    createTime: dataset.createTime,
    updateTime: dataset.updateTime,
  };
};

const getByName = (name: string) =>
  name.length === 0
    ? Effect.undefined
    : aiplatform.getDatasets({ name }).pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const listDatasets = (project: string, region: string) =>
  Effect.forEach(listLocations(region), (location) =>
    listDatasetsAt(parentOf(project, location)),
  ).pipe(Effect.map((pages) => pages.flat()));

const listDatasetsAt = (parent: string) =>
  aiplatform.listProjectsLocationsDatasets
    .pages({
      parent,
      pageSize: 100,
    })
    .pipe(
      Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
      Stream.filter((dataset) => hasAlchemyLabelKeys(dataset.labels)),
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
      Effect.catchTag("NotFound", () => Effect.succeed([])),
    );

const findOwned = (id: string, project: string, region: string, hinted?: string) =>
  Effect.gen(function* () {
    if (hinted !== undefined && hinted.length > 0) {
      const existing = yield* getByName(hinted);
      if (existing !== undefined) return existing;
    }
    const owned = yield* listDatasets(project, region);
    for (const dataset of owned) {
      if (yield* hasAlchemyLabels(id, tagRecord(dataset.labels))) {
        return dataset;
      }
    }
    return undefined as aiplatform.GoogleCloudAiplatformV1Dataset | undefined;
  });

const waitUntilExists = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((dataset) =>
      dataset
        ? Effect.succeed(dataset)
        : Effect.fail(new DatasetNotResolved({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.DatasetNotResolved",
      times: 8,
      schedule: Schedule.spaced("1 second"),
    }),
  );

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((dataset) =>
      dataset === undefined
        ? Effect.void
        : Effect.fail(new DatasetStillExists({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.DatasetStillExists",
      times: 10,
      schedule: Schedule.spaced("2 seconds"),
    }),
  );

export const DatasetProvider = () =>
  Provider.succeed(Dataset, {
    stables: ["name", "datasetId", "project", "location", "metadataSchemaUri", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      const env = yield* GcpEnvironment.current;
      if (!isResolved(news)) return undefined;
      const previousId = olds?.datasetId ?? output?.datasetId;
      const nextId = news.datasetId ?? previousId;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
      const nextLocation = normalizeLocation(
        news.location ?? olds?.location ?? output?.location,
        env.region,
      );
      const previousSchema = schemaUriOf(olds?.metadataSchemaUri ?? output?.metadataSchemaUri);
      const nextSchema = schemaUriOf(
        news.metadataSchemaUri ?? olds?.metadataSchemaUri ?? output?.metadataSchemaUri,
      );
      const previousKey = olds?.encryptionSpec?.kmsKeyName ?? output?.kmsKeyName ?? "";
      const nextKey = news.encryptionSpec?.kmsKeyName ?? previousKey;
      const replace =
        (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
        previousLocation !== nextLocation ||
        previousSchema !== nextSchema ||
        previousKey !== nextKey;
      if (!replace) return undefined;
      return {
        action: "replace" as const,
        deleteFirst:
          previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
      };
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const existing = yield* findOwned(id, env.project, env.region, output?.name);
      if (existing === undefined) {
        if (output?.name) return undefined;
        // Dataset ids are server-assigned numbers; only a caller-supplied
        // id can be looked up by name.
        const datasetId = olds?.datasetId;
        if (datasetId === undefined) return undefined;
        const location = normalizeLocation(olds?.location ?? output?.location, env.region);
        const named = yield* getByName(resourceName(env.project, location, datasetId));
        if (named === undefined) return undefined;
        const attrs = toAttrs(named, env.project);
        return (yield* hasAlchemyLabels(id, tagRecord(named.labels))) ? attrs : Unowned(attrs);
      }
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const datasets = yield* listDatasets(env.project, env.region);
        return datasets.map((dataset) => toAttrs(dataset, env.project));
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const datasetId = yield* toId(id, news.datasetId, output?.datasetId);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };
      const displayName = news.displayName ?? datasetId;
      const metadataSchemaUri = schemaUriOf(news.metadataSchemaUri);

      let current = yield* findOwned(id, env.project, env.region, output?.name);
      if (current === undefined && news.datasetId !== undefined) {
        current = yield* getByName(resourceName(env.project, location, news.datasetId));
      }

      if (current === undefined) {
        const created = yield* aiplatform
          .createProjectsLocationsDatasets({
            parent: parentOf(env.project, location),
            body: {
              displayName,
              description: news.description,
              labels: desiredLabels,
              metadataSchemaUri,
              metadata: news.metadata ?? {},
              encryptionSpec: news.encryptionSpec,
              modelReference: news.modelReference,
              savedQueries: news.savedQueries,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        if (created !== undefined) {
          const done = yield* waitForOperation(created);
          const createdName =
            resourceNameFromOperation(done) ??
            (yield* findOwned(id, env.project, env.region))?.name;
          if (createdName !== undefined && createdName.length > 0) {
            current = yield* waitUntilExists(createdName);
          }
        }
        if (current === undefined) {
          current = yield* findOwned(id, env.project, env.region);
        }
      }

      if (current === undefined) {
        return yield* new DatasetNotResolved({
          resourceName: output?.name ?? resourceName(env.project, location, datasetId),
        });
      }

      const name = current.name ?? "";
      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      const labelsChanged = upsert.length > 0 || removed.length > 0;
      const displayNameChanged = (current.displayName ?? "") !== displayName;
      const descriptionChanged = (current.description ?? "") !== (news.description ?? "");

      if (labelsChanged || displayNameChanged || descriptionChanged) {
        current = yield* aiplatform.patchProjectsLocationsDatasets({
          name,
          updateMask: [
            labelsChanged ? "labels" : undefined,
            displayNameChanged ? "display_name" : undefined,
            descriptionChanged ? "description" : undefined,
          ]
            .filter((field): field is string => field !== undefined)
            .join(","),
          body: {
            name,
            labels: desiredLabels,
            displayName,
            description: news.description,
          },
        });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      const existing = yield* getByName(output.name);
      if (existing === undefined) return;
      const operation = yield* aiplatform
        .deleteDatasets({ name: output.name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(output.name);
    }),
  });