sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ DataPipelines/ internal.ts

import * as datapipelines from "@distilled.cloud/gcp/datapipelines_v1";
import { createPhysicalName } from "alchemy/PhysicalName";
import { tagRecord } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import { alchemyLabelKeys, hasAlchemyLabels, stripInternalLabels } from "../Labels.ts";

export const DEFAULT_TYPE = "PIPELINE_TYPE_BATCH";
export const DEFAULT_STATE = "STATE_ACTIVE";
export const MAX_ID_LENGTH = 63;
export const MAX_DISPLAY_NAME_LENGTH = 64;
// Data Pipelines rejects the `-` location wildcard on list with a bare
// INVALID_ARGUMENT, so enumerate regions explicitly.
export const LIST_LOCATIONS = [
  "us-central1",
  "us-east1",
  "us-west1",
  "europe-west1",
  "asia-east1",
] as const;

export const lastSegment = (value: string) => {
  const trimmed = value.replace(/\/+$/, "");
  const parts = trimmed.split("/");
  return parts[parts.length - 1] || trimmed;
};

export const sameText = (left: string | undefined, right: string | undefined) =>
  (left ?? "") === (right ?? "");

// Key-order-insensitive: the API echoes maps (e.g. launch `parameters`) in
// its own order, which would otherwise read as a workload change.
const canonical = (value: unknown): unknown =>
  Array.isArray(value)
    ? value.map(canonical)
    : value !== null && typeof value === "object"
      ? Object.fromEntries(
          Object.entries(value)
            .filter(([, entry]) => entry !== undefined)
            .sort(([a], [b]) => a.localeCompare(b))
            .map(([key, entry]) => [key, canonical(entry)]),
        )
      : value;

export const jsonEqual = (left: unknown, right: unknown) =>
  JSON.stringify(canonical(left ?? null)) === JSON.stringify(canonical(right ?? null));

export const updateMaskOf = (...fields: Array<string | undefined>) =>
  fields.filter((field): field is string => field !== undefined).join(",");

export const normalizeLocation = (location: string | undefined, fallback: string) =>
  lastSegment(location ?? fallback).toLowerCase();

export const locationParent = (project: string, location: string) =>
  `projects/${project}/locations/${location}`;

export const resourceName = (project: string, location: string, pipelineId: string) =>
  `${locationParent(project, location)}/pipelines/${pipelineId}`;

export const parseName = (name: string, fallbackLocation: string) => {
  const parts = name.split("/").filter((part) => part.length > 0);
  const projectsAt = parts.lastIndexOf("projects");
  const locationsAt = parts.lastIndexOf("locations");
  const pipelinesAt = parts.lastIndexOf("pipelines");
  return {
    project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
    location:
      locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : fallbackLocation,
    pipelineId:
      pipelinesAt >= 0 && parts[pipelinesAt + 1] ? parts[pipelinesAt + 1]! : lastSegment(name),
    parent:
      pipelinesAt > 0
        ? parts.slice(0, pipelinesAt).join("/")
        : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
  };
};

export const rfc1035 = (name: string, maxLength = MAX_ID_LENGTH, fallback = "pipeline"): string => {
  let next = name
    .toLowerCase()
    .replace(/[^a-z0-9-]/g, "-")
    .replace(/-+/g, "-")
    .replace(/^-+|-+$/g, "");
  if (!/^[a-z]/.test(next)) next = `p${next}`;
  next = next.slice(0, maxLength).replace(/-+$/g, "");
  if (next.length === 0) return fallback.slice(0, maxLength);
  if (!/[a-z0-9]$/.test(next)) {
    next = `${next.slice(0, maxLength - 1)}0`;
  }
  return next.slice(0, maxLength);
};

export const sanitizeDisplayName = (value: string) => {
  let next = value
    .replace(/[^A-Za-z0-9_-]/g, "_")
    .replace(/_+/g, "_")
    .replace(/^_+|_+$/g, "")
    .slice(0, MAX_DISPLAY_NAME_LENGTH);
  if (next.length === 0) return "pipeline";
  if (!/^[A-Za-z]/.test(next)) {
    next = `p${next}`.slice(0, MAX_DISPLAY_NAME_LENGTH);
  }
  return next;
};

export const toPipelineId = (
  id: string,
  requested: string | undefined,
  existing: string | undefined,
) =>
  Effect.gen(function* () {
    if (requested !== undefined && requested.length > 0) {
      return rfc1035(requested);
    }
    if (existing !== undefined && existing.length > 0) return existing;
    return rfc1035(
      yield* createPhysicalName({
        id,
        maxLength: MAX_ID_LENGTH,
        lowercase: true,
      }),
    );
  });

export const toDisplayName = (requested: string | undefined, fallback: string) =>
  requested !== undefined && requested.length > 0
    ? sanitizeDisplayName(requested)
    : sanitizeDisplayName(fallback);

export const stringMapOf = (
  value: Record<string, string | undefined> | null | undefined,
): Record<string, string> => tagRecord(value);

export const userSources = (
  sources: Record<string, string | undefined> | null | undefined,
): Record<string, string> => stripInternalLabels(stringMapOf(sources));

export const ownershipSources = (
  labels: Record<string, string>,
  user?: Record<string, string>,
): Record<string, string> => ({
  ...user,
  [alchemyLabelKeys.stack]: labels[alchemyLabelKeys.stack] ?? "x",
  [alchemyLabelKeys.stage]: labels[alchemyLabelKeys.stage] ?? "x",
  [alchemyLabelKeys.id]: labels[alchemyLabelKeys.id] ?? "x",
});

export const hasOwnershipMarker = (
  sources: Record<string, string | undefined> | null | undefined,
) => Object.keys(sources ?? {}).some((key) => key.startsWith("alchemy-"));

export const ownedByAlchemy = (
  id: string,
  sources: Record<string, string | undefined> | null | undefined,
) => hasAlchemyLabels(id, stringMapOf(sources));

export const scheduleKey = (
  info: datapipelines.GoogleCloudDatapipelinesV1ScheduleSpec | undefined,
) => ({
  schedule: info?.schedule ?? "",
  timeZone: info?.timeZone ?? "",
});

export const replaceOnIdentity = (input: {
  previousId?: string;
  nextId?: string;
  previousLocation?: string;
  nextLocation?: string;
  previousType?: string;
  nextType?: string;
}) => {
  if (
    input.previousId !== undefined &&
    input.nextId !== undefined &&
    input.previousId !== input.nextId
  ) {
    return { action: "replace" as const, deleteFirst: false };
  }
  if (
    input.previousLocation !== undefined &&
    input.nextLocation !== undefined &&
    input.previousLocation !== input.nextLocation
  ) {
    return { action: "replace" as const, deleteFirst: false };
  }
  if (
    input.previousType !== undefined &&
    input.nextType !== undefined &&
    input.previousType !== input.nextType
  ) {
    return { action: "replace" as const, deleteFirst: false };
  }
  return undefined;
};

export const getPipeline = (name: string) =>
  name.length === 0
    ? Effect.undefined
    : datapipelines
        .getProjectsLocationsPipelines({ name })
        .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

export const ignoreMissing = <A, R>(
  effect: Effect.Effect<A, datapipelines.DeleteProjectsLocationsPipelinesError, R>,
) => effect.pipe(Effect.catchTag("NotFound", () => Effect.void));

const emptyList = <A>() => Effect.succeed([] as A[]);

export const listPipelinesAt = (parent: string) =>
  datapipelines.listProjectsLocationsPipelines
    .pages({
      parent,
      pageSize: 200,
    })
    .pipe(
      Stream.flatMap((page) => Stream.fromIterable(page.pipelines ?? [])),
      Stream.runCollect,
      Effect.map((chunk) => Array.from(chunk)),
      Effect.catchTag("NotFound", () =>
        emptyList<datapipelines.GoogleCloudDatapipelinesV1Pipeline>(),
      ),
    );

export const listOwnedPipelines = (project: string, region: string) =>
  Effect.gen(function* () {
    const pages = yield* Effect.forEach(
      [...new Set([region, ...LIST_LOCATIONS])],
      (location) => listPipelinesAt(locationParent(project, location)),
      { concurrency: 2 },
    );
    const seen = new Set<string>();
    const owned: datapipelines.GoogleCloudDatapipelinesV1Pipeline[] = [];
    for (const row of pages.flat()) {
      const name = row.name ?? "";
      if (name.length > 0 && seen.has(name)) continue;
      if (!hasOwnershipMarker(row.pipelineSources)) continue;
      if (name.length > 0) seen.add(name);
      owned.push(row);
    }
    return owned;
  });

export const findOwnedPipeline = (id: string, project: string, region: string, name?: string) =>
  Effect.gen(function* () {
    const existing = yield* getPipeline(name ?? "");
    if (existing !== undefined) return existing;
    const rows = yield* listOwnedPipelines(project, region);
    for (const row of rows) {
      if (yield* ownedByAlchemy(id, row.pipelineSources)) {
        return row;
      }
    }
    return undefined;
  });