sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ TPU/ QueuedResource.ts

import * as tpu from "@distilled.cloud/gcp/tpu_v2";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  encodeDescription,
  hasAlchemyLabelMap,
  hasOwnershipMarker,
  normalizeLocation,
  parseDescription,
  parseName,
  replaceOnIdentity,
  ResourceNotResolved,
  toPhysicalId,
  waitForOperation,
  waitUntilExists,
  waitUntilGone,
} from "./internal.ts";
import { toNodeBody, type NodeProps } from "./Node.ts";

export type MultisliceParams = {
  /**
   * Prefix for generated node ids (`np` → `np-0`, `np-1`, …). Defaults to
   * the queued resource id.
   */
  nodeIdPrefix?: string;
  /**
   * Number of nodes to provision. Must be greater than 1.
   */
  nodeCount?: number;
};

export type NodeSpec = {
  /**
   * Parent location
   * (`projects/{project}/locations/{location}`). Defaults to the queued
   * resource parent.
   */
  parent?: string;
  /**
   * Unqualified node id when requesting a single node. Mutually exclusive
   * with `multisliceParams`.
   */
  nodeId?: string;
  /**
   * Multislice request. Mutually exclusive with `nodeId`.
   */
  multisliceParams?: MultisliceParams;
  /**
   * Node configuration. Alchemy ownership labels are merged into
   * `node.labels` (QueuedResource has no labels field of its own).
   */
  node?: NodeProps;
};

export type QueueingPolicy = {
  /** Absolute time after which the request fails if still unfulfilled. */
  validUntilTime?: string;
  /** Relative duration after which resources may be created. */
  validAfterDuration?: string;
  /** Absolute interval during which resources may be created. */
  validInterval?: {
    /** Inclusive start (RFC3339). */
    startTime?: string;
    /** Exclusive end (RFC3339). */
    endTime?: string;
  };
  /** Absolute time after which resources may be created. */
  validAfterTime?: string;
  /** Relative duration after which the request fails if still unfulfilled. */
  validUntilDuration?: string;
};

export type Guaranteed = {
  /**
   * Minimum duration the resources must be allocated (`3600s`, …).
   */
  minDuration?: string;
};

export type QueuedResourceProps = {
  /**
   * Queued resource id (the `{queuedResource}` segment of
   * `projects/{project}/locations/{location}/queuedResources/{queuedResource}`).
   * If omitted, a unique RFC1035 name is generated from the stack, stage,
   * and logical id. Immutable — changing it replaces the resource.
   */
  queuedResourceId?: string;
  /**
   * Zone (`us-central1-c`, …). Immutable — changing it replaces the
   * resource.
   * @default "us-central1-c"
   */
  location?: string;
  /**
   * Reservation to provision in
   * (`projects/{project}/locations/{zone}/reservations/{reservation}`).
   * Immutable.
   */
  reservationName?: string;
  /**
   * Request the Spot tier. Immutable.
   * @default false
   */
  spot?: boolean;
  /**
   * Guaranteed tier. Immutable.
   */
  guaranteed?: Guaranteed;
  /**
   * When the request may be fulfilled. Immutable.
   */
  queueingPolicy?: QueueingPolicy;
  /**
   * TPU node specification. Omit to request a default v2-8 node with the
   * standard runtime. Alchemy ownership is stamped on each nested node's
   * labels and description (QueuedResource has no labels API).
   */
  nodeSpec?: NodeSpec[];
  /**
   * Delete running nodes belonging to this request before deleting the
   * queued resource.
   * @default true
   */
  forceDestroy?: boolean;
};

export type QueuedResource = Resource<
  "GCP.TPU.QueuedResource",
  QueuedResourceProps,
  {
    /** Full resource name `projects/{project}/locations/{location}/queuedResources/{queuedResource}`. */
    name: string;
    /** Queued resource id (last path segment). */
    queuedResourceId: string;
    /** Project id. */
    project: string;
    /** Zone id. */
    location: string;
    /** Reservation name, if any. */
    reservationName: string | undefined;
    /** Whether the Spot tier is set. */
    spot: boolean;
    /** Guaranteed tier configuration. */
    guaranteed: Guaranteed | undefined;
    /** Queueing policy. */
    queueingPolicy: QueueingPolicy | undefined;
    /** Node specification currently stored on the request. */
    nodeSpec: NodeSpec[];
    /** Server-reported state (`ACCEPTED`, `WAITING_FOR_RESOURCES`, …). */
    state: string | undefined;
    /** Who initiated the current state (`USER`, `SERVICE`). */
    stateInitiator: string | undefined;
    /** Error that caused `FAILED`, if any. */
    failedError: string | undefined;
    /** RFC3339 creation timestamp. */
    createTime: string | undefined;
  },
  never,
  Providers
>;

/**
 * A Cloud TPU queued resource request. The request sits in a queue until
 * capacity is available, then provisions the requested TPU node(s).
 *
 * Queued resources have no labels field — Alchemy stamps ownership onto
 * each nested `nodeSpec.node` labels and description so `list` / nuke can
 * find them. The resource is create-only: changing identity or the node
 * spec replaces it.
 *
 * ### Creating a Queued Resource
 * **Example:** Default v2-8 request
 * ```typescript
 * const request = yield* GCP.TPU.QueuedResource("Trainer", {});
 * ```
 *
 * **Example:** Spot request with an explicit node spec
 * ```typescript
 * const request = yield* GCP.TPU.QueuedResource("Trainer", {
 *   location: "us-central1-c",
 *   spot: true,
 *   nodeSpec: [{
 *     nodeId: "app-tpu",
 *     node: {
 *       acceleratorType: "v2-8",
 *       runtimeVersion: "tpu-ubuntu2204-base",
 *       labels: { env: "prod" },
 *     },
 *   }],
 * });
 * ```
 *
 * @resource
 * @category TPU
 */
export const QueuedResource = Resource<QueuedResource>("GCP.TPU.QueuedResource");

const resourceName = (project: string, location: string, queuedResourceId: string) =>
  `projects/${project}/locations/${location}/queuedResources/${queuedResourceId}`;

const parentOf = (project: string, location: string) => `projects/${project}/locations/${location}`;

const intervalOf = (interval: tpu.Interval | QueueingPolicy["validInterval"] | undefined) => {
  if (interval === undefined) return undefined;
  return { startTime: interval.startTime, endTime: interval.endTime };
};

const queueingOf = (
  policy: tpu.QueueingPolicy | QueueingPolicy | undefined,
): QueueingPolicy | undefined => {
  if (policy === undefined) return undefined;
  return {
    validUntilTime: policy.validUntilTime,
    validAfterDuration: policy.validAfterDuration,
    validInterval: intervalOf(policy.validInterval),
    validAfterTime: policy.validAfterTime,
    validUntilDuration: policy.validUntilDuration,
  };
};

const queueingKey = (policy: QueueingPolicy | undefined) =>
  JSON.stringify({
    validUntilTime: policy?.validUntilTime ?? "",
    validAfterDuration: policy?.validAfterDuration ?? "",
    validIntervalStart: policy?.validInterval?.startTime ?? "",
    validIntervalEnd: policy?.validInterval?.endTime ?? "",
    validAfterTime: policy?.validAfterTime ?? "",
    validUntilDuration: policy?.validUntilDuration ?? "",
  });

const guaranteedOf = (
  guaranteed: tpu.Guaranteed | Guaranteed | undefined,
): Guaranteed | undefined => {
  if (guaranteed === undefined) return undefined;
  return { minDuration: guaranteed.minDuration };
};

const networkTagsOf = (node: tpu.Node | NodeProps) => {
  if ("networkTags" in node && node.networkTags) {
    return [...node.networkTags];
  }
  if ("tags" in node && node.tags) {
    return [...node.tags];
  }
  return undefined;
};

const stringRecordOf = (map: Record<string, string | undefined> | undefined) =>
  map
    ? Object.fromEntries(
        Object.entries(map).filter((entry): entry is [string, string] => entry[1] !== undefined),
      )
    : undefined;

const nodeSpecOf = (spec: tpu.NodeSpec | NodeSpec, parent: string): NodeSpec => ({
  parent: spec.parent ?? parent,
  nodeId: spec.nodeId,
  multisliceParams: spec.multisliceParams
    ? {
        nodeIdPrefix: spec.multisliceParams.nodeIdPrefix,
        nodeCount: spec.multisliceParams.nodeCount,
      }
    : undefined,
  node: spec.node
    ? {
        description: spec.node.description,
        labels: stringRecordOf(spec.node.labels),
        metadata: stringRecordOf(spec.node.metadata),
        networkTags: networkTagsOf(spec.node),
        runtimeVersion: spec.node.runtimeVersion,
        acceleratorType: spec.node.acceleratorType,
        acceleratorConfig: spec.node.acceleratorConfig,
        cidrBlock: spec.node.cidrBlock,
        serviceAccount: spec.node.serviceAccount,
        networkConfig: spec.node.networkConfig,
        networkConfigs: spec.node.networkConfigs,
        bootDiskConfig: spec.node.bootDiskConfig,
        dataDisks: spec.node.dataDisks,
        schedulingConfig: spec.node.schedulingConfig,
        shieldedInstanceConfig: spec.node.shieldedInstanceConfig,
      }
    : undefined,
});

const nodeSpecKey = (spec: NodeSpec) =>
  JSON.stringify({
    parent: spec.parent ?? "",
    nodeId: spec.nodeId ?? "",
    prefix: spec.multisliceParams?.nodeIdPrefix ?? "",
    count: spec.multisliceParams?.nodeCount ?? "",
    runtime: spec.node?.runtimeVersion ?? "",
    accelerator: spec.node?.acceleratorType ?? "",
    topology: spec.node?.acceleratorConfig?.topology ?? "",
    cidr: spec.node?.cidrBlock ?? "",
  });

const specsKey = (specs: readonly NodeSpec[]) => JSON.stringify(specs.map(nodeSpecKey));

const nestedNodes = (resource: tpu.QueuedResource) => resource.tpu?.nodeSpec ?? [];

const nestedOwnershipLabels = (resource: tpu.QueuedResource) => {
  const labels: Record<string, string> = {};
  for (const spec of nestedNodes(resource)) {
    Object.assign(labels, tagRecord(spec.node?.labels));
    Object.assign(labels, parseDescription(spec.node?.description).labels);
  }
  return labels;
};

const hasNestedOwnership = (resource: tpu.QueuedResource) => {
  for (const spec of nestedNodes(resource)) {
    if (hasAlchemyLabelMap(spec.node?.labels)) return true;
    if (hasOwnershipMarker(spec.node?.description)) return true;
  }
  return false;
};

const desiredSpecs = (
  news: QueuedResourceProps,
  parent: string,
  desiredLabels: Record<string, string>,
): tpu.NodeSpec[] => {
  const specs =
    news.nodeSpec !== undefined && news.nodeSpec.length > 0
      ? news.nodeSpec
      : [{ node: {} as NodeProps }];
  return specs.map((spec) => {
    const node = spec.node ?? {};
    const description = encodeDescription(desiredLabels, node.description);
    const labels = { ...toLabels(node.labels), ...desiredLabels };
    return {
      parent: spec.parent ?? parent,
      nodeId: spec.nodeId,
      multisliceParams: spec.multisliceParams,
      node: toNodeBody({ ...node, labels }, labels, { description }),
    };
  });
};

const toAttrs = (resource: tpu.QueuedResource, project: string) => {
  const name = resource.name ?? "";
  const parsed = parseName(name, "queuedResources");
  const parent = parentOf(parsed.project || project, parsed.location);
  return {
    name,
    queuedResourceId: parsed.id,
    project: parsed.project || project,
    location: parsed.location,
    reservationName: resource.reservationName,
    spot: resource.spot !== undefined,
    guaranteed: guaranteedOf(resource.guaranteed),
    queueingPolicy: queueingOf(resource.queueingPolicy),
    nodeSpec: (resource.tpu?.nodeSpec ?? []).map((spec) => {
      const mapped = nodeSpecOf(spec, parent);
      if (mapped.node?.description !== undefined) {
        mapped.node = {
          ...mapped.node,
          description: parseDescription(mapped.node.description).description,
          labels: mapped.node.labels
            ? Object.fromEntries(
                Object.entries(mapped.node.labels).filter(([key]) => !key.startsWith("alchemy-")),
              )
            : undefined,
        };
      }
      return mapped;
    }),
    state: resource.state?.state,
    stateInitiator: resource.state?.stateInitiator,
    failedError: resource.state?.failedData?.error?.message,
    createTime: resource.createTime,
  };
};

const isPlaceholder = (resource: tpu.QueuedResource) => {
  const name = resource.name ?? "";
  return (
    name.length === 0 || name.endsWith("/queuedResources/-") || name.endsWith("/queuedResources/")
  );
};

const getByName = (name: string) =>
  tpu
    .getProjectsLocationsQueuedResources({ name })
    .pipe(Effect.catchTag("NotFound", () => Effect.undefined));

const specIdentityChanged = (
  previous: QueuedResourceProps | { nodeSpec?: NodeSpec[] } | undefined,
  next: QueuedResourceProps,
  parent: string,
) => {
  const previousSpecs = (previous?.nodeSpec ?? []).map((spec) => nodeSpecOf(spec, parent));
  const nextSpecs =
    next.nodeSpec !== undefined && next.nodeSpec.length > 0
      ? next.nodeSpec.map((spec) => nodeSpecOf(spec, parent))
      : previousSpecs;
  if (previousSpecs.length === 0) return false;
  return specsKey(previousSpecs) !== specsKey(nextSpecs);
};

export const QueuedResourceProvider = () =>
  Provider.succeed(QueuedResource, {
    stables: ["name", "queuedResourceId", "project", "location", "createTime"],

    diff: Effect.fn(function* ({ news, olds, output }) {
      if (!isResolved(news)) return undefined;

      const previousId = olds?.queuedResourceId ?? output?.queuedResourceId;
      const nextId = news.queuedResourceId ?? previousId;
      const previousLocation = normalizeLocation(olds?.location ?? output?.location);
      const nextLocation = normalizeLocation(news.location ?? output?.location);
      const parent = `projects/_/locations/${nextLocation}`;
      const previousReservation = olds?.reservationName ?? output?.reservationName ?? "";
      const nextReservation = news.reservationName ?? previousReservation;
      const previousSpot = olds?.spot ?? output?.spot ?? false;
      const nextSpot = news.spot ?? previousSpot;
      const previousGuaranteed =
        guaranteedOf(olds?.guaranteed ?? output?.guaranteed)?.minDuration ?? "";
      const nextGuaranteed =
        guaranteedOf(news.guaranteed ?? olds?.guaranteed ?? output?.guaranteed)?.minDuration ??
        previousGuaranteed;
      const previousQueueing = queueingKey(
        queueingOf(olds?.queueingPolicy ?? output?.queueingPolicy),
      );
      const nextQueueing = queueingKey(
        queueingOf(news.queueingPolicy ?? olds?.queueingPolicy ?? output?.queueingPolicy),
      );

      return replaceOnIdentity({
        previousId,
        nextId,
        previousLocation,
        nextLocation,
        extra:
          previousReservation !== nextReservation ||
          previousSpot !== nextSpot ||
          previousGuaranteed !== nextGuaranteed ||
          previousQueueing !== nextQueueing ||
          specIdentityChanged(olds ?? output, news, parent),
      });
    }),

    read: Effect.fn(function* ({ id, olds, output }) {
      const env = yield* GcpEnvironment.current;
      const queuedResourceId = yield* toPhysicalId(
        id,
        olds?.queuedResourceId,
        output?.queuedResourceId,
      );
      const location = normalizeLocation(olds?.location ?? output?.location);
      const name = output?.name ?? resourceName(env.project, location, queuedResourceId);
      const existing = yield* getByName(name);
      if (existing === undefined) return undefined;
      const attrs = toAttrs(existing, env.project);
      const nested = nestedOwnershipLabels(existing);
      return (yield* hasAlchemyLabels(id, nested)) ? attrs : Unowned(attrs);
    }),

    list: () =>
      Effect.gen(function* () {
        const env = yield* GcpEnvironment.current;
        return yield* tpu.listProjectsLocationsQueuedResources
          .pages({
            parent: `projects/${env.project}/locations/-`,
            pageSize: 1000,
          })
          .pipe(
            Stream.flatMap((page) => Stream.fromIterable(page.queuedResources ?? [])),
            Stream.filter((resource) => !isPlaceholder(resource) && hasNestedOwnership(resource)),
            Stream.map((resource) => toAttrs(resource, env.project)),
            Stream.runCollect,
            Effect.map((chunk) => Array.from(chunk)),
            Effect.catchTag("NotFound", () => Effect.succeed([])),
          );
      }),

    reconcile: Effect.fn(function* ({ id, news, output }) {
      const env = yield* GcpEnvironment.current;
      const queuedResourceId = yield* toPhysicalId(
        id,
        news.queuedResourceId,
        output?.queuedResourceId,
      );
      const location = normalizeLocation(news.location ?? output?.location);
      const parent = parentOf(env.project, location);
      const name = resourceName(env.project, location, queuedResourceId);
      const desiredLabels = yield* createInternalLabels(id);

      let current = yield* getByName(output?.name ?? name);

      if (current === undefined) {
        const created = yield* tpu
          .createProjectsLocationsQueuedResources({
            parent,
            queuedResourceId,
            body: {
              reservationName: news.reservationName,
              spot: news.spot === true ? {} : undefined,
              guaranteed: news.guaranteed,
              queueingPolicy: news.queueingPolicy,
              tpu: { nodeSpec: desiredSpecs(news, parent, desiredLabels) },
            },
          })
          .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
        if (created !== undefined) {
          yield* waitForOperation(created);
        }
        current = yield* waitUntilExists(getByName(name), name);
      }

      if (current === undefined) {
        return yield* new ResourceNotResolved({ resourceName: name });
      }

      return toAttrs(current, env.project);
    }),

    delete: Effect.fn(function* ({ output, olds }) {
      const force = olds?.forceDestroy !== false;
      const operation = yield* tpu
        .deleteProjectsLocationsQueuedResources({
          name: output.name,
          force,
        })
        .pipe(
          Effect.catchTag("NotFound", () => Effect.undefined),
          Effect.retry({
            while: (error) => error._tag === "Conflict",
            times: 8,
            schedule: Schedule.spaced("5 seconds"),
          }),
        );
      if (operation !== undefined) {
        yield* waitForOperation(operation, { notFoundOk: true });
      }
      yield* waitUntilGone(getByName(output.name), output.name);
    }),
  });