sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ AIPlatform/ FeatureOnlineStoresFeatureView.ts

import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  expandParent,
  fingerprint,
  hasAlchemyLabelMap,
  lastSegment,
  normalizeLocation,
  parseResourceName,
  specifiedEquals,
  toPhysicalSnake,
  userLabels,
} from "./helpers.ts";
import { listLocations } from "./names.ts";
import { waitForOperation } from "./operations.ts";

const MAX_NAME_LENGTH = 60;

export type FeatureViewBigQuerySource = {
  /** BigQuery view URI materialized on each sync. */
  uri?: string;
  /** Columns used as entity ids. */
  entityIdColumns?: string[];
};

export type FeatureViewFeatureRegistrySource = {
  /** Feature groups and feature ids to sync. */
  featureGroups?: Array<{
    featureGroupId?: string;
    featureIds?: string[];
  }>;
  /** Parent project number of the Feature Groups. */
  projectNumber?: string;
};

export type FeatureViewVertexRagSource = {
  /** BigQuery view/table URI. */
  uri?: string;
  /** RAG corpus id. */
  ragCorpusId?: string;
};

export type FeatureViewSyncConfig = {
  /** Cron schedule. */
  cron?: string;
  /** Continuous sync. */
  continuous?: boolean;
};

export type FeatureViewIndexConfig = {
  embeddingColumn?: string;
  embeddingDimension?: number;
  crowdingColumn?: string;
  filterColumns?: string[];
  distanceMeasureType?:
    | aiplatform.GoogleCloudAiplatformV1FeatureViewIndexConfigDistanceMeasureTypeEnum
    | (string & {});
  treeAhConfig?: { leafNodeEmbeddingCount?: string };
  bruteForceConfig?: Record<string, never>;
};

export type FeatureOnlineStoresFeatureViewProps = {
  /**
   * Parent Feature Online Store resource name or id. Immutable.
   */
  featureOnlineStore: string;
  /**
   * Feature View id. Valid characters `[a-z0-9_]`, must start with a
   * letter, max 60. Immutable.
   */
  featureViewId?: string;
  /**
   * Region used when `featureOnlineStore` is a bare id.
   * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
   */
  location?: string;
  /**
   * User labels. Alchemy ownership labels are merged in automatically.
   */
  labels?: Record<string, string>;
  /**
   * BigQuery source. Mutually exclusive with `featureRegistrySource`
   * and `vertexRagSource`.
   */
  bigQuerySource?: FeatureViewBigQuerySource;
  /**
   * Feature Registry source.
   */
  featureRegistrySource?: FeatureViewFeatureRegistrySource;
  /**
   * Vertex RAG source.
   */
  vertexRagSource?: FeatureViewVertexRagSource;
  /**
   * Sync schedule.
   */
  syncConfig?: FeatureViewSyncConfig;
  /**
   * Vector index config.
   */
  indexConfig?: FeatureViewIndexConfig;
  /**
   * Optimized-store replica config.
   */
  optimizedConfig?: {
    automaticResources?: { minReplicaCount?: number; maxReplicaCount?: number };
  };
  /**
   * Service agent type used during data sync.
   */
  serviceAgentType?:
    | aiplatform.GoogleCloudAiplatformV1FeatureViewServiceAgentTypeEnum
    | (string & {});
  /**
   * Run an on-demand sync immediately at create.
   * @default false
   */
  runSyncImmediately?: boolean;
};

export type FeatureOnlineStoresFeatureView = Resource<
  "GCP.AIPlatform.FeatureOnlineStoresFeatureView",
  FeatureOnlineStoresFeatureViewProps,
  {
    /** Full resource name. */
    name: string;
    /** Feature view id. */
    featureViewId: string;
    /** Parent online store resource name. */
    featureOnlineStore: string;
    /** Parent online store id. */
    featureOnlineStoreId: string;
    /** Project id. */
    project: string;
    /** Location id. */
    location: string;
    /** User labels (Alchemy ownership labels stripped). */
    labels: Record<string, string>;
    /** BigQuery source. */
    bigQuerySource: aiplatform.GoogleCloudAiplatformV1FeatureViewBigQuerySource | undefined;
    /** Feature Registry source. */
    featureRegistrySource:
      | aiplatform.GoogleCloudAiplatformV1FeatureViewFeatureRegistrySource
      | undefined;
    /** Vertex RAG source. */
    vertexRagSource: aiplatform.GoogleCloudAiplatformV1FeatureViewVertexRagSource | undefined;
    /** Sync config. */
    syncConfig: aiplatform.GoogleCloudAiplatformV1FeatureViewSyncConfig | undefined;
    /** Vector index config. */
    indexConfig: aiplatform.GoogleCloudAiplatformV1FeatureViewIndexConfig | undefined;
    /** Optimized replica config. */
    optimizedConfig: aiplatform.GoogleCloudAiplatformV1FeatureViewOptimizedConfig | undefined;
    /** Service agent type. */
    serviceAgentType:
      | aiplatform.GoogleCloudAiplatformV1FeatureViewServiceAgentTypeEnum
      | (string & {})
      | undefined;
    /** Unique service account email. */
    serviceAccountEmail: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
    /** RFC3339 last-update timestamp. */
    updateTime: string | undefined;
    /** Server etag. */
    etag: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Feature View — the serving projection of a Feature Online Store.
 *
 * Parent store, view id, and location are identity. Labels, sources, sync
 * schedule, and optimized replica counts update in place.
 *
 * ### Creating a Feature View
 * **Example:** Registry-backed view
 * ```typescript
 * const view = yield* GCP.AIPlatform.FeatureOnlineStoresFeatureView("Users", {
 *   featureOnlineStore: store.name,
 *   featureRegistrySource: {
 *     featureGroups: [{ featureGroupId: group.featureGroupId, featureIds: ["age"] }],
 *   },
 *   syncConfig: { cron: "0 * * * *" },
 * });
 * ```
 *
 * @resource
 * @category AIPlatform
 */
export const FeatureOnlineStoresFeatureView = Resource<FeatureOnlineStoresFeatureView>(
  "GCP.AIPlatform.FeatureOnlineStoresFeatureView",
);

export class FeatureOnlineStoresFeatureViewNotResolved extends Data.TaggedError(
  "GCP.AIPlatform.FeatureOnlineStoresFeatureViewNotResolved",
)<{
  resourceName: string;
}> {}

export class FeatureOnlineStoresFeatureViewStillExists extends Data.TaggedError(
  "GCP.AIPlatform.FeatureOnlineStoresFeatureViewStillExists",
)<{
  resourceName: string;
}> {}

const parentOf = (project: string, location: string, store: string) =>
  expandParent(store, project, location, "featureOnlineStores");

const resourceName = (parent: string, viewId: string) => `${parent}/featureViews/${viewId}`;

const toAttrs = (view: aiplatform.GoogleCloudAiplatformV1FeatureView, project: string) => {
  const name = view.name ?? "";
  const parsed = parseResourceName(name, "featureViews");
  const store = parseResourceName(parsed.parent, "featureOnlineStores");
  return {
    name,
    featureViewId: parsed.id,
    featureOnlineStore: parsed.parent,
    featureOnlineStoreId: store.id,
    project: parsed.project || project,
    location: parsed.location,
    labels: userLabels(view.labels),
    bigQuerySource: view.bigQuerySource,
    featureRegistrySource: view.featureRegistrySource,
    vertexRagSource: view.vertexRagSource,
    syncConfig: view.syncConfig,
    indexConfig: view.indexConfig,
    optimizedConfig: view.optimizedConfig,
    serviceAgentType: view.serviceAgentType,
    serviceAccountEmail: view.serviceAccountEmail,
    createTime: view.createTime,
    updateTime: view.updateTime,
    etag: view.etag,
  };
};

const getByName = (name: string) =>
  aiplatform
    .getProjectsLocationsFeatureOnlineStoresFeatureViews({ name })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const waitUntilExists = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((view) =>
      view
        ? Effect.succeed(view)
        : Effect.fail(new FeatureOnlineStoresFeatureViewNotResolved({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.FeatureOnlineStoresFeatureViewNotResolved",
      times: 8,
      schedule: Schedule.spaced("1 second"),
    }),
  );

const waitUntilGone = (name: string) =>
  getByName(name).pipe(
    Effect.flatMap((view) =>
      view === undefined
        ? Effect.void
        : Effect.fail(new FeatureOnlineStoresFeatureViewStillExists({ resourceName: name })),
    ),
    Effect.retry({
      while: (error) => error._tag === "GCP.AIPlatform.FeatureOnlineStoresFeatureViewStillExists",
      times: 10,
      schedule: Schedule.spaced("2 seconds"),
    }),
  );

const listViewsUnder = (parent: string, project: string) =>
  aiplatform.listProjectsLocationsFeatureOnlineStoresFeatureViews
    .pages({ parent, pageSize: 100 })
    .pipe(
      Stream.flatMap((page) => Stream.fromIterable(page.featureViews ?? [])),
      Stream.filter((view) => hasAlchemyLabelMap(view.labels)),
      Stream.map((view) => toAttrs(view, project)),
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
      Effect.catchTag("NotFound", () => Effect.succeed([])),
    );

export const FeatureOnlineStoresFeatureViewProvider = () =>
  Provider.succeed(FeatureOnlineStoresFeatureView, {
    stables: [
      "name",
      "featureViewId",
      "featureOnlineStore",
      "featureOnlineStoreId",
      "project",
      "location",
      "createTime",
    ],

    diff: Effect.fn(function* ({ news, olds, output }) {
      const env = yield* GcpEnvironment.current;
      if (!isResolved(news)) return undefined;
      const previousId = olds?.featureViewId ?? output?.featureViewId;
      const nextId = news.featureViewId ?? previousId;
      const previousParent = olds?.featureOnlineStore ?? output?.featureOnlineStore ?? "";
      const nextParent = news.featureOnlineStore ?? previousParent;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
      const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
      const parentChanged =
        previousParent.length > 0 && lastSegment(nextParent) !== lastSegment(previousParent);
      const replace =
        (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
        parentChanged ||
        previousLocation !== nextLocation;
      if (!replace) return undefined;
      return { action: "replace" as const, deleteFirst: false };
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(olds?.location ?? output?.location, env.region);
      const parentRef = olds?.featureOnlineStore ?? output?.featureOnlineStore;
      // A create interrupted before its parent resolved has nothing to find.
      if (output?.name === undefined && typeof parentRef !== "string") {
        return undefined;
      }
      const parent = parentOf(env.project, location, parentRef ?? "");
      const viewId = yield* toPhysicalSnake(
        id,
        olds?.featureViewId,
        output?.featureViewId,
        MAX_NAME_LENGTH,
      );
      const name = output?.name ?? resourceName(parent, viewId);
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        const stores = yield* Stream.fromIterable(listLocations(env.region)).pipe(
          Stream.flatMap((location) =>
            aiplatform.listProjectsLocationsFeatureOnlineStores.pages({
              parent: `projects/${env.project}/locations/${location}`,
              pageSize: 100,
            }),
          ),
          Stream.flatMap((page) => Stream.fromIterable(page.featureOnlineStores ?? [])),
          Stream.filter((store) => hasAlchemyLabelMap(store.labels)),
          Stream.runCollect,
          Effect.map((chunk) => Array.from(chunk)),
          Effect.catchTag("NotFound", () => Effect.succeed([])),
        );
        const nested = yield* Effect.forEach(
          stores,
          (store) => (store.name ? listViewsUnder(store.name, env.project) : Effect.succeed([])),
          { concurrency: 4 },
        );
        return nested.flat();
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const location = normalizeLocation(news.location ?? output?.location, env.region);
      const parent = parentOf(env.project, location, news.featureOnlineStore);
      const viewId = yield* toPhysicalSnake(
        id,
        news.featureViewId,
        output?.featureViewId,
        MAX_NAME_LENGTH,
      );
      const name = resourceName(parent, viewId);
      const desiredLabels = {
        ...toLabels(news.labels),
        ...(yield* createInternalLabels(id)),
      };

      let current = yield* getByName(output?.name ?? name);

      if (current === undefined) {
        const created = yield* aiplatform
          .createProjectsLocationsFeatureOnlineStoresFeatureViews({
            parent,
            featureViewId: viewId,
            runSyncImmediately: news.runSyncImmediately === true,
            body: {
              labels: desiredLabels,
              bigQuerySource: news.bigQuerySource,
              featureRegistrySource: news.featureRegistrySource,
              vertexRagSource: news.vertexRagSource,
              syncConfig: news.syncConfig,
              indexConfig: news.indexConfig,
              optimizedConfig: news.optimizedConfig,
              serviceAgentType: news.serviceAgentType,
            },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        if (created !== undefined) {
          yield* waitForOperation(created);
        }
        current = yield* waitUntilExists(name);
      }

      if (current === undefined) {
        return yield* new FeatureOnlineStoresFeatureViewNotResolved({ resourceName: name });
      }

      const observedLabels = tagRecord(current.labels);
      const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
      const labelsChanged = upsert.length > 0 || removed.length > 0;
      const bqChanged =
        news.bigQuerySource !== undefined &&
        !specifiedEquals(news.bigQuerySource, current.bigQuerySource);
      const registryChanged =
        news.featureRegistrySource !== undefined &&
        !specifiedEquals(news.featureRegistrySource, current.featureRegistrySource);
      const syncChanged =
        news.syncConfig !== undefined &&
        fingerprint(current.syncConfig) !== fingerprint(news.syncConfig);
      const optimizedChanged =
        news.optimizedConfig !== undefined &&
        !specifiedEquals(news.optimizedConfig, current.optimizedConfig);
      const agentChanged =
        news.serviceAgentType !== undefined &&
        (current.serviceAgentType ?? "") !== news.serviceAgentType;

      if (
        labelsChanged ||
        bqChanged ||
        registryChanged ||
        syncChanged ||
        optimizedChanged ||
        agentChanged
      ) {
        const updateMask = [
          labelsChanged ? "labels" : undefined,
          bqChanged ? "big_query_source" : undefined,
          registryChanged ? "feature_registry_source" : undefined,
          syncChanged ? "sync_config" : undefined,
          optimizedChanged ? "optimized_config.automatic_resources" : undefined,
          agentChanged ? "service_agent_type" : undefined,
        ].filter((field): field is string => field !== undefined);

        const patched = yield* aiplatform
          .patchProjectsLocationsFeatureOnlineStoresFeatureViews({
            name,
            updateMask: updateMask.join(","),
            body: {
              name,
              labels: desiredLabels,
              bigQuerySource: news.bigQuerySource,
              featureRegistrySource: news.featureRegistrySource,
              syncConfig: news.syncConfig,
              optimizedConfig: news.optimizedConfig,
              serviceAgentType: news.serviceAgentType,
            },
          })
          .pipe(
            Effect.retry({
              while: (error) => error._tag === "Conflict",
              times: 8,
              schedule: Schedule.spaced("5 seconds"),
            }),
          );
        yield* waitForOperation(patched);
        current = yield* waitUntilExists(name);
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output }) {
      const operation = yield* aiplatform
        .deleteProjectsLocationsFeatureOnlineStoresFeatureViews({
          name: output.name,
        })
        .pipe(
          Effect.catchTag("NotFound", () => Effect.undefined),
          Effect.retry({
            while: (error) => error._tag === "Conflict",
            times: 8,
            schedule: Schedule.spaced("5 seconds"),
          }),
        );
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(output.name);
    }),
  });