sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Fly/ fixtures/ idle-cadence-readiness.ts

import { expect } from "@alchemy.run/test";
import type { ScratchStack } from "@alchemy.run/test/Alchemy";
import { scratchStack } from "@alchemy.run/test/Core";
import type * as machines from "@distilled.cloud/fly-io/machines";
import * as Effect from "effect/Effect";
import * as HttpClient from "effect/http/HttpClient";
import * as Layer from "effect/Layer";
import { throughProxy, transportProxy, type TransportEvent } from "./transport.ts";

const metadataKeys = [
  "alchemy.stack",
  "alchemy.stage",
  "alchemy.id",
  "alchemy.type",
  "alchemy.replica",
  "alchemy.instance",
  "alchemy.fqn",
  "alchemy.base-name",
  "alchemy.generation",
  "alchemy.workload",
  "alchemy.sequence",
  "alchemy.count",
  "alchemy.image",
  "alchemy.phase",
  "alchemy.deployment-protocol",
  "alchemy.readiness-roles",
  "alchemy.readiness-role",
  "alchemy.predecessors",
  "alchemy.idle-policy-restored",
  "alchemy.checked-instance",
  "alchemy.min-secrets-version",
];

const record = (value: unknown): Record<string, unknown> =>
  typeof value === "object" && value !== null && !Array.isArray(value)
    ? (value as Record<string, unknown>)
    : {};
const string = (value: unknown) => (typeof value === "string" ? value : undefined);
const metadataOf = (value: unknown) => {
  const metadata = record(value);
  return Object.fromEntries(
    metadataKeys.flatMap((key) =>
      typeof metadata[key] === "string" ? [[key, metadata[key] as string]] : [],
    ),
  );
};
const servicesOf = (value: unknown) =>
  Array.isArray(value)
    ? value.map((item) => {
        const service = record(item);
        return {
          protocol: string(service.protocol),
          port: typeof service.internal_port === "number" ? service.internal_port : undefined,
          autostop:
            service.autostop === true
              ? "stop"
              : service.autostop === "stop" || service.autostop === "suspend"
                ? service.autostop
                : "off",
          autostart: typeof service.autostart === "boolean" ? service.autostart : undefined,
          floor:
            typeof service.min_machines_running === "number"
              ? service.min_machines_running
              : undefined,
          ...(Array.isArray(service.checks)
            ? {
                checks: service.checks.map((value) => {
                  const check = record(value);
                  return {
                    type: string(check.type),
                    port: typeof check.port === "number" ? check.port : undefined,
                    interval: string(check.interval),
                    timeout: string(check.timeout),
                    gracePeriod: string(check.grace_period),
                    method: string(check.method),
                    protocol: string(check.protocol),
                  };
                }),
              }
            : {}),
        };
      })
    : [];

export interface ReadinessEvent extends TransportEvent {
  metadata?: Record<string, string>;
  metadataOnly?: boolean;
  skipLaunch?: boolean;
  services?: ReturnType<typeof servicesOf>;
  serviceConfig?: string;
}

/** Only these two excluded suites install this observer; bodies and headers are never retained. */
export const readinessProxy = () =>
  transportProxy().pipe(
    Effect.map((proxy) => ({ ...proxy, readiness: [] as ReadinessEvent[] })),
    Effect.tap((proxy) =>
      Effect.addFinalizer(() =>
        Effect.forEach(
          Array.from({ length: Math.ceil(proxy.readiness.length / 50) }, (_, index) => index),
          (index) =>
            Effect.logInfo("Idle/cadence readiness journal", {
              offset: index * 50,
              events: proxy.readiness.slice(index * 50, (index + 1) * 50),
            }),
          { discard: true },
        ),
      ),
    ),
  );

type ReadinessProxy = Effect.Success<ReturnType<typeof readinessProxy>>;

export const readinessActor = (
  parent: ScratchStack,
  title: string,
  file: string,
  proxy: ReadinessProxy,
  beforeRequest?: (event: ReadinessEvent) => Effect.Effect<void>,
) =>
  Effect.sync(() => {
    const observer = Layer.effect(
      HttpClient.HttpClient,
      Effect.gen(function* () {
        const client = yield* HttpClient.HttpClient;
        return HttpClient.transform(client, (response, request) =>
          Effect.gen(function* () {
            const event = yield* Effect.sync(() => {
              const path = new URL(request.url).pathname;
              if (!/\/apps\/[^/]+\/machines(?:\/|$)/.test(path)) return undefined;
              const event: ReadinessEvent = {
                sequence: proxy.readiness.length,
                stage: "request",
                method: request.method,
                path,
                machineId: path.match(/\/machines\/([^/]+)/)?.[1],
              };
              if (request.body._tag === "Uint8Array") {
                try {
                  const body = record(JSON.parse(new TextDecoder().decode(request.body.body)));
                  const config = record(body.config);
                  const metadata = record(body.metadata ?? config.metadata);
                  event.metadata = metadataOf(metadata);
                  event.phase = event.metadata["alchemy.phase"];
                  event.skipLaunch =
                    typeof body.skip_launch === "boolean" ? body.skip_launch : undefined;
                  event.metadataOnly =
                    Object.keys(body).length === 1 &&
                    "metadata" in body &&
                    Object.keys(metadata).every(
                      (key) => metadataKeys.includes(key) && typeof metadata[key] === "string",
                    );
                } catch {
                  // An unreadable body cannot qualify for the bookkeeping exemption.
                }
              }
              proxy.readiness.push(event);
              return event;
            });
            if (event && beforeRequest) yield* beforeRequest(event);
            const result = yield* response;
            if (event) {
              const value =
                result.status >= 200 &&
                result.status < 300 &&
                /\/machines(?:\/[^/]+)?$/.test(event.path)
                  ? yield* result.json.pipe(Effect.catch(() => Effect.succeed(undefined)))
                  : undefined;
              yield* Effect.sync(() => {
                const machine = record(value);
                const config = record(machine.config);
                const observed: ReadinessEvent = {
                  ...event,
                  // Receipt here is after the unmodified proxy forwarded the response.
                  stage: "forwarded",
                  status: result.status,
                };
                if (typeof machine.id === "string") {
                  observed.machineId = machine.id;
                  observed.instanceId = string(machine.instance_id);
                  observed.state = string(machine.state);
                  observed.cordoned =
                    typeof machine.cordoned === "boolean" ? machine.cordoned : undefined;
                  observed.digest = string(record(machine.image_ref).digest);
                  observed.metadata = metadataOf(config.metadata);
                  observed.phase = observed.metadata["alchemy.phase"];
                  observed.services = servicesOf(config.services);
                  observed.serviceConfig = JSON.stringify(observed.services);
                  observed.checks = Array.isArray(machine.checks)
                    ? machine.checks.map((item) => {
                        const check = record(item);
                        return {
                          name: string(check.name),
                          status: string(check.status),
                        };
                      })
                    : [];
                }
                proxy.readiness.push(observed);
              });
            }
            return result;
          }),
        );
      }),
    ).pipe(Layer.provideMerge(throughProxy(() => proxy.url)));
    const actor = scratchStack({ providers: observer, stage: parent.stage }, title, file);
    expect(actor.name).toBe(parent.name);
    expect(actor.stage).toBe(parent.stage);
    expect(actor.state).not.toBe(parent.state);
    return actor;
  });

export const retires = (event: TransportEvent, ids: readonly string[]) =>
  event.stage === "request" &&
  ids.includes(event.machineId!) &&
  (event.path.endsWith("/cordon") ||
    event.path.endsWith("/stop") ||
    event.path.endsWith("/suspend") ||
    (event.path.endsWith("/metadata") && event.phase === "retiring") ||
    (event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path)));

const identity = (metadata: Record<string, string> | undefined) =>
  Object.fromEntries(
    Object.entries(metadata ?? {}).filter(
      ([key]) => key !== "alchemy.phase" && key !== "alchemy.checked-instance",
    ),
  );

export const readinessChecksPassing = (
  checks: ReadinessEvent["checks"],
  expected: readonly string[],
) => {
  if (!checks || expected.length === 0 || new Set(expected).size !== expected.length) return false;
  const allowed = new Set([
    ...expected,
    ...expected
      .filter((name) => name.startsWith("servicecheck-"))
      .map((name) => `bg_deployments_compat-${name.slice("servicecheck-".length)}`),
  ]);
  const reported = new Set(checks.map((check) => check.name));
  return (
    reported.size === checks.length &&
    expected.every((name) => reported.has(name)) &&
    checks.every(
      (check) => check.name !== undefined && allowed.has(check.name) && check.status === "passing",
    )
  );
};

const sameMetadata = (left: ReadinessEvent, right: ReadinessEvent) =>
  left.method === "PATCH" &&
  right.method === "PATCH" &&
  left.path.endsWith("/metadata") &&
  left.path === right.path &&
  left.metadataOnly === true &&
  right.metadataOnly === true &&
  left.metadata !== undefined &&
  right.metadata !== undefined &&
  JSON.stringify(Object.entries(left.metadata).sort()) ===
    JSON.stringify(Object.entries(right.metadata).sort());

/** Fresh GET receipt proves observation order, not the remote check sample's timestamp. */
export const assertReadinessCommit = (
  events: readonly ReadinessEvent[],
  priorIds: readonly string[],
  candidates: machines.Machine[],
  runningSlots: readonly number[],
  checkNames: readonly string[],
  allowIdle: boolean,
) => {
  const retirement =
    priorIds.length > 0 ? events.findIndex((event) => retires(event, priorIds)) : events.length;
  expect(retirement).toBeGreaterThan(0);
  const ordered = [...candidates].sort(
    (a, b) =>
      Number(a.config?.metadata?.["alchemy.replica"]) -
      Number(b.config?.metadata?.["alchemy.replica"]),
  );
  const ids = ordered.map((machine) => machine.id!);
  const creates = events
    .filter(
      (event) =>
        event.stage === "forwarded" &&
        event.method === "POST" &&
        event.path.endsWith("/machines") &&
        ids.includes(event.machineId!),
    )
    .map((event) => event.sequence);
  const mutations = events.filter(
    (event) =>
      event.stage === "request" &&
      event.method !== "GET" &&
      !event.path.endsWith("/lease") &&
      (ids.includes(event.machineId!) || creates.includes(event.sequence)),
  );
  // Phase stamps go through Fly's `PATCH /machines/{id}/metadata` (formerly PUT).
  const isCommit = (event: ReadinessEvent) =>
    event.method === "PATCH" &&
    event.path.endsWith("/metadata") &&
    event.phase === "active" &&
    event.metadataOnly === true;
  const isValidating = (event: ReadinessEvent) =>
    event.method === "PATCH" &&
    event.path.endsWith("/metadata") &&
    event.phase === "validating" &&
    event.metadataOnly === true;
  const successful = (event: ReadinessEvent) => event.status! >= 200 && event.status! < 300;
  const operations: {
    request: ReadinessEvent;
    machineId: string;
    completed: number;
  }[] = [];
  for (const request of mutations) {
    const receipts = events.filter(
      (event) =>
        event.sequence === request.sequence &&
        event.stage === "forwarded" &&
        event.method === request.method &&
        event.path === request.path,
    );
    expect(receipts).toHaveLength(1);
    const receipt = receipts[0]!;
    const completed = events.indexOf(receipt);
    const requested = events.indexOf(request);
    expect(completed).toBeGreaterThan(requested);
    expect(completed).toBeLessThan(retirement);
    const machineId = request.machineId ?? receipt.machineId!;
    if (request.path.endsWith("/start") && receipt.status === 412) {
      const candidate = ordered.find((machine) => machine.id === machineId)!;
      const commit = events.findIndex((event) => isCommit(event) && event.machineId === machineId);
      expect(
        events.some(
          (event, index) =>
            index > completed &&
            index < commit &&
            event.stage === "forwarded" &&
            event.method === "GET" &&
            successful(event) &&
            event.machineId === machineId &&
            event.state === "started" &&
            event.instanceId === candidate.instance_id,
        ),
      ).toBe(true);
      continue;
    }
    const previous = operations.findLast((operation) => operation.machineId === machineId);
    // Only settled, retryable metadata failures may join an equivalent retry.
    if (
      previous &&
      !successful(events[previous.completed]!) &&
      sameMetadata(previous.request, request)
    ) {
      expect([429, 500, 502, 503, 504]).toContain(events[previous.completed]!.status);
      expect(previous.completed).toBeLessThan(requested);
      previous.completed = completed;
    } else {
      operations.push({ request, machineId, completed });
    }
  }
  for (const operation of operations) {
    expect(successful(events[operation.completed]!)).toBe(true);
  }
  const commits = operations.filter((operation) => isCommit(operation.request));
  expect(commits).toHaveLength(ordered.length);
  expect(commits.map((operation) => operation.machineId).sort()).toEqual([...ids].sort());
  expect(commits.at(-1)?.machineId).toBe(ordered[0]!.id);
  const firstCommit = events.indexOf(commits[0]!.request);
  expect(firstCommit).toBeGreaterThan(0);
  const invalidating = operations.filter((operation) => !isCommit(operation.request));
  expect(invalidating.length).toBeGreaterThan(0);
  const reset = Math.max(...invalidating.map((operation) => operation.completed));
  expect(reset).toBeLessThan(firstCommit);
  const zeroCommit = events.indexOf(commits.at(-1)!.request);
  for (const operation of commits.slice(0, -1)) {
    expect(operation.completed).toBeLessThan(zeroCommit);
  }

  for (const [slot, machine] of ordered.entries()) {
    const id = machine.id!;
    const run = runningSlots.includes(slot);
    const commit = commits.find((operation) => operation.machineId === id)!.request;
    const finalMetadata = metadataOf(machine.config?.metadata);
    const finalServices = servicesOf(machine.config?.services);
    if (allowIdle) {
      expect(finalServices.length).toBeGreaterThan(0);
      expect(finalServices.every((service) => service.autostop !== "off")).toBe(true);
    }
    expect(finalMetadata["alchemy.replica"]).toBe(String(slot));
    expect(finalMetadata["alchemy.readiness-role"]).toBe(run ? "run" : "idle");
    expect(finalMetadata["alchemy.readiness-roles"]).toBe(
      ordered.map((_, index) => (runningSlots.includes(index) ? "run" : "idle")).join(","),
    );
    expect(finalMetadata["alchemy.phase"]).toBe("active");
    expect(finalMetadata["alchemy.idle-policy-restored"]).toBe("true");
    expect(commit.metadata).toEqual(finalMetadata);
    expect(machine.cordoned).toBe(false);
    for (const key of [
      "alchemy.stack",
      "alchemy.stage",
      "alchemy.id",
      "alchemy.type",
      "alchemy.instance",
      "alchemy.fqn",
      "alchemy.generation",
      "alchemy.workload",
      "alchemy.image",
    ]) {
      expect(finalMetadata[key]).toBeDefined();
    }
    const observations = events.flatMap((event, index) => {
      if (
        index >= firstCommit ||
        event.stage !== "forwarded" ||
        event.method !== "GET" ||
        !event.path.endsWith(`/machines/${id}`) ||
        event.machineId !== id ||
        event.status !== 200
      )
        return [];
      const requested = events.findIndex(
        (request) => request.stage === "request" && request.sequence === event.sequence,
      );
      return requested >= 0 && requested < index ? [{ event, index, requested }] : [];
    });
    const pending = observations.findLast(({ requested }) => requested > reset);
    expect(pending).toBeDefined();
    if (!pending) continue;
    expect(pending.event.phase).toBe("validating");
    expect(pending.event.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
    expect(pending.event.instanceId).toBe(machine.instance_id);
    let proof = pending;
    if (run && pending.event.state !== "started") {
      expect(allowIdle).toBe(true);
      expect(["stopped", "suspended"]).toContain(pending.event.state);
      const targetReset = Math.max(
        -1,
        ...invalidating
          .filter((operation) => operation.machineId === id && !isValidating(operation.request))
          .map((operation) => operation.completed),
      );
      const restored = observations.findLast(
        ({ event, requested, index }) =>
          requested > targetReset &&
          index < pending.index &&
          ["promoting", "validating"].includes(event.phase ?? "") &&
          event.metadata?.["alchemy.idle-policy-restored"] === "true" &&
          event.instanceId === pending.event.instanceId &&
          event.state === "started" &&
          readinessChecksPassing(event.checks, checkNames),
      );
      expect(restored).toBeDefined();
      if (!restored) continue;
      proof = restored;
    } else if (run) {
      expect(readinessChecksPassing(proof.event.checks, checkNames)).toBe(true);
    } else {
      expect(["created", "stopped", "suspended"]).toContain(pending.event.state);
    }
    expect(proof.event.instanceId).toBeDefined();
    expect(proof.event.digest).toBe(machine.image_ref?.digest);
    expect(proof.event.services).toEqual(finalServices);
    expect(proof.event.cordoned).toBe(false);
    expect(identity(proof.event.metadata)).toEqual(identity(finalMetadata));
    expect(commit.metadata).toEqual({
      ...pending.event.metadata,
      "alchemy.phase": "active",
      ...(run ? { "alchemy.checked-instance": proof.event.instanceId } : {}),
    });
    if (run) {
      expect(commit.metadata?.["alchemy.checked-instance"]).toBe(proof.event.instanceId);
      expect(allowIdle ? ["started", "stopped", "suspended"] : ["started"]).toContain(
        machine.state,
      );
    } else {
      expect(["created", "stopped", "suspended"]).toContain(machine.state);
      for (const created of events.filter(
        (event) =>
          event.stage === "forwarded" &&
          event.method === "POST" &&
          event.path.endsWith("/machines") &&
          event.machineId === id,
      )) {
        expect(created.skipLaunch).toBe(true);
      }
      expect(
        events.some(
          (event) =>
            event.machineId === id &&
            ((event.stage === "request" && event.path.endsWith("/start")) ||
              (event.stage === "forwarded" && event.state === "started")),
        ),
      ).toBe(false);
    }
    // Idle proof survives only the unchanged pending stamp and final commit.
    for (const operation of invalidating.filter(
      (operation) => operation.machineId === id && operation.completed > proof.index,
    )) {
      expect(isValidating(operation.request)).toBe(true);
      expect(events.indexOf(operation.request)).toBeGreaterThan(proof.index);
      expect(operation.request.metadata).toEqual({
        ...proof.event.metadata,
        "alchemy.phase": "validating",
      });
    }
    for (const later of events
      .slice(proof.index + 1)
      .filter(
        (event) =>
          event.stage === "forwarded" &&
          event.machineId === id &&
          event.status! >= 200 &&
          event.status! < 300 &&
          event.path.endsWith(`/machines/${id}`),
      )) {
      expect(later.instanceId).toBe(proof.event.instanceId);
      expect(later.digest).toBe(proof.event.digest);
      expect(later.services).toEqual(finalServices);
      expect(identity(later.metadata)).toEqual(identity(finalMetadata));
      expect(later.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
      expect(["promoting", "validating", "active"]).toContain(later.phase);
      expect(later.cordoned).toBe(false);
      if (run) {
        expect(allowIdle ? ["started", "stopped", "suspended"] : ["started"]).toContain(
          later.state,
        );
        if (later.state === "started" && events.indexOf(later) < firstCommit) {
          expect(readinessChecksPassing(later.checks, checkNames)).toBe(true);
        }
        if (later.phase === "active") {
          expect(later.metadata?.["alchemy.checked-instance"]).toBe(proof.event.instanceId);
        }
      }
    }
  }
};