sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ Bigtable/ operations.ts

import * as bigtable from "@distilled.cloud/gcp/bigtableadmin_v2";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { waitForOperation as waitForGcpOperation } from "../Operation.ts";

export const DEFAULT_ZONE = "us-central1-b";
export const DEFAULT_STORAGE = "HDD";
export const DEFAULT_SERVE_NODES = 1;
export const DEFAULT_INSTANCE_TYPE = "PRODUCTION";
export const DEFAULT_EDITION = "ENTERPRISE";
export const DEFAULT_CLUSTER_ID = "cluster";
export const MAX_INSTANCE_ID_LENGTH = 33;
export const MIN_INSTANCE_ID_LENGTH = 6;
export const MAX_CLUSTER_ID_LENGTH = 30;
export const MIN_CLUSTER_ID_LENGTH = 6;
export const MAX_APP_PROFILE_ID_LENGTH = 50;
export const MAX_TABLE_ID_LENGTH = 50;
export const MAX_BACKUP_ID_LENGTH = 50;
export const MAX_LOGICAL_VIEW_ID_LENGTH = 50;
export const MAX_MATERIALIZED_VIEW_ID_LENGTH = 50;
export const MAX_AUTHORIZED_VIEW_ID_LENGTH = 50;
export const MAX_SCHEMA_BUNDLE_ID_LENGTH = 50;

export const lastSegment = (value: string) => {
  const trimmed = value.replace(/\/+$/, "");
  const parts = trimmed.split("/");
  return parts[parts.length - 1] || trimmed;
};

export const rfc1035 = (name: string, maxLength: number, minLength = 1): string => {
  let next = name
    .toLowerCase()
    .replace(/[^a-z0-9-]/g, "-")
    .replace(/-+/g, "-")
    .replace(/^-+|-+$/g, "");
  if (!/^[a-z]/.test(next)) next = `b${next}`;
  next = next.slice(0, maxLength).replace(/-+$/g, "");
  if (next.length === 0) next = "bigtable";
  if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, maxLength - 1)}0`;
  if (next.length < minLength) {
    next = `${next}${"x".repeat(minLength)}`.slice(0, minLength);
  }
  return next.slice(0, maxLength);
};

export const toPhysicalId = (
  id: string,
  explicit: string | undefined,
  existing: string | undefined,
  maxLength: number,
  minLength = 1,
) =>
  Effect.gen(function* () {
    if (explicit !== undefined) return explicit;
    if (existing !== undefined) return existing;
    return rfc1035(
      yield* createPhysicalName({
        id,
        maxLength,
        lowercase: true,
      }),
      maxLength,
      minLength,
    );
  });

export const instanceName = (project: string, instanceId: string) =>
  `projects/${project}/instances/${instanceId}`;

export const clusterName = (project: string, instanceId: string, clusterId: string) =>
  `${instanceName(project, instanceId)}/clusters/${clusterId}`;

export const appProfileName = (project: string, instanceId: string, appProfileId: string) =>
  `${instanceName(project, instanceId)}/appProfiles/${appProfileId}`;

export const tableName = (project: string, instanceId: string, tableId: string) =>
  `${instanceName(project, instanceId)}/tables/${tableId}`;

export const backupName = (
  project: string,
  instanceId: string,
  clusterId: string,
  backupId: string,
) => `${clusterName(project, instanceId, clusterId)}/backups/${backupId}`;

export const logicalViewName = (project: string, instanceId: string, logicalViewId: string) =>
  `${instanceName(project, instanceId)}/logicalViews/${logicalViewId}`;

export const materializedViewName = (
  project: string,
  instanceId: string,
  materializedViewId: string,
) => `${instanceName(project, instanceId)}/materializedViews/${materializedViewId}`;

export const authorizedViewName = (
  project: string,
  instanceId: string,
  tableId: string,
  authorizedViewId: string,
) => `${tableName(project, instanceId, tableId)}/authorizedViews/${authorizedViewId}`;

export const schemaBundleName = (
  project: string,
  instanceId: string,
  tableId: string,
  schemaBundleId: string,
) => `${tableName(project, instanceId, tableId)}/schemaBundles/${schemaBundleId}`;

export const parseResourceName = (name: string) => {
  const parts = name.split("/").filter((part) => part.length > 0);
  const indexOf = (segment: string) => parts.lastIndexOf(segment);
  const after = (segment: string) => {
    const at = indexOf(segment);
    return at >= 0 && parts[at + 1] ? parts[at + 1]! : "";
  };
  const project = after("projects");
  const instanceId = after("instances");
  const clusterId = after("clusters");
  const tableId = after("tables");
  return {
    project,
    instanceId,
    clusterId,
    appProfileId: after("appProfiles"),
    tableId,
    backupId: after("backups"),
    logicalViewId: after("logicalViews"),
    materializedViewId: after("materializedViews"),
    authorizedViewId: after("authorizedViews"),
    schemaBundleId: after("schemaBundles"),
    instance: project && instanceId ? instanceName(project, instanceId) : "",
    cluster: project && instanceId && clusterId ? clusterName(project, instanceId, clusterId) : "",
    table: project && instanceId && tableId ? tableName(project, instanceId, tableId) : "",
  };
};

export const instanceNameOf = (project: string, instance: string) => {
  if (instance.includes("/instances/")) {
    const parsed = parseResourceName(instance);
    return instanceName(parsed.project || project, parsed.instanceId);
  }
  return instanceName(project, lastSegment(instance));
};

export const instanceIdOf = (value: string) =>
  value.includes("/instances/") ? parseResourceName(value).instanceId : lastSegment(value);

export const clusterIdOf = (value: string) =>
  value.includes("/clusters/") ? parseResourceName(value).clusterId : lastSegment(value);

export const tableIdOf = (value: string) =>
  value.includes("/tables/") ? parseResourceName(value).tableId : lastSegment(value);

export const clusterNameOf = (project: string, instance: string, cluster: string) => {
  if (cluster.includes("/clusters/")) {
    const parsed = parseResourceName(cluster);
    return clusterName(
      parsed.project || project,
      parsed.instanceId || instanceIdOf(instance),
      parsed.clusterId,
    );
  }
  return clusterName(project, instanceIdOf(instance), lastSegment(cluster));
};

export const tableNameOf = (project: string, instance: string, table: string) => {
  if (table.includes("/tables/")) {
    const parsed = parseResourceName(table);
    return tableName(
      parsed.project || project,
      parsed.instanceId || instanceIdOf(instance),
      parsed.tableId,
    );
  }
  return `${instanceNameOf(project, instance)}/tables/${lastSegment(table)}`;
};

export const clusterLocation = (project: string, location: string) => {
  if (location.includes("/locations/")) {
    const zone = lastSegment(location);
    const parsed = parseResourceName(location);
    return `projects/${parsed.project || project}/locations/${zone}`;
  }
  return `projects/${project}/locations/${lastSegment(location).toLowerCase()}`;
};

export const zoneOf = (location: string | undefined) =>
  lastSegment(location ?? DEFAULT_ZONE).toLowerCase();

export const hasAlchemyPrefix = (labels: Record<string, string | undefined> | null | undefined) =>
  Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));

/**
 * Wait for a Bigtable admin operation. Cluster resizes and materialized
 * views are the slowest (several minutes). With `alreadyExistsOk`, an
 * ALREADY_EXISTS (code 6) result counts as success (create race).
 */
export const waitForOperation = (
  operation: bigtable.Operation,
  options?: { alreadyExistsOk?: boolean },
) =>
  waitForGcpOperation(operation, (name) => bigtable.getOperations({ name }), {
    budget: "20 minutes",
  }).pipe(
    Effect.catchIf(
      (error) =>
        options?.alreadyExistsOk === true &&
        error._tag === "GCP.OperationFailed" &&
        error.code === 6,
      () => Effect.succeed(operation),
    ),
  );

export const getInstanceByName = (name: string) =>
  bigtable.getProjectsInstances({ name }).pipe(Effect.catchTag("NotFound", () => Effect.undefined));

export const collectPages = <Page, A, E, R>(
  pages: Stream.Stream<Page, E, R>,
  items: (page: Page) => readonly A[] | undefined,
) =>
  pages.pipe(
    Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
    Stream.runCollect,
    Effect.map((chunk) => Array.from(chunk)),
  );

export const listAlchemyInstances = (project: string) =>
  collectPages(
    bigtable.listProjectsInstances.pages({ parent: `projects/${project}` }),
    (page) => page.instances,
  ).pipe(
    Effect.map((instances) => instances.filter((instance) => hasAlchemyPrefix(instance.labels))),
  );

export const parentOwned = (instanceNameValue: string) =>
  getInstanceByName(instanceNameValue).pipe(
    Effect.map((instance) => (instance === undefined ? true : hasAlchemyPrefix(instance.labels))),
  );

export const listAlchemyTables = (project: string) =>
  Effect.gen(function* () {
    const instances = (yield* listAlchemyInstances(project)).filter(
      (instance): instance is bigtable.Instance & { name: string } =>
        typeof instance.name === "string" && instance.name.length > 0,
    );
    const pages = yield* Effect.forEach(
      instances,
      (instance) =>
        collectPages(
          bigtable.listProjectsInstancesTables.pages({
            parent: instance.name,
            pageSize: 1000,
          }),
          (page) => page.tables,
        ).pipe(
          // The instance was deleted between the two list calls.
          Effect.catchTag("NotFound", () => Effect.succeed([] as bigtable.Table[])),
        ),
      { concurrency: 4 },
    );
    return pages.flat();
  });