sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ Fly/ BlueGreen.test.ts

import { randomBytes, createHash } from "node:crypto";
import * as Docker from "@alchemy.run/docker";
import * as Fly from "@alchemy.run/fly";
import { AppDeletionAmbiguous } from "@alchemy.run/fly/App";
import { readinessRoles, DeploymentRecoveryAmbiguous } from "@alchemy.run/fly/bluegreen";
import { makeMachineLeases } from "@alchemy.run/fly/leases";
import { type MachineContainer, type MachineProps } from "@alchemy.run/fly/Machine";
import { alchemyMetadataKeys as keys } from "@alchemy.run/fly/Metadata";
import {
  waitHealthy,
  autostopMode,
  checksPassing,
  ensureStarted,
  observeReplicaSet,
  predecessorShutdown,
  deleteReplicaSet,
  ReplicaNotCreated,
  ReplicaRetirementIncomplete,
  retireMachines,
  retireMachine,
} from "@alchemy.run/fly/replicas";
import { ReplicaChecksNotPassing } from "@alchemy.run/fly/replicas";
import { expect, assert, it, describe } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import type { ScratchStack } from "@alchemy.run/test/Alchemy";
import * as TestCore from "@alchemy.run/test/Core";
import { scratchStack, withProviders } from "@alchemy.run/test/Core";
import {
  GatewayTimeout,
  HTTP_STATUS_MAP,
  RETRYABLE_HTTP_STATUSES,
} from "@distilled.cloud/core/errors";
import { Credentials } from "@distilled.cloud/fly-io/Credentials";
import { GatewayTimeout as GatewayTimeoutErrors } from "@distilled.cloud/fly-io/Errors";
import * as machines from "@distilled.cloud/fly-io/machines";
import { type Machine } from "@distilled.cloud/fly-io/machines";
import * as Retry from "@distilled.cloud/fly-io/Retry";
import * as Alchemy from "alchemy";
import { DestroyError } from "alchemy/Apply";
import { localState, makeLocalState } from "alchemy/State/LocalState";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as ConfigProvider from "effect/ConfigProvider";
import * as Data from "effect/Data";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as FetchHttpClient from "effect/http/FetchHttpClient";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Path from "effect/Path";
import * as Redacted from "effect/Redacted";
import * as Ref from "effect/Ref";
import * as Result from "effect/Result";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import { engineActor } from "./fixtures/actors.ts";
import { dropCompletedCreate } from "./fixtures/bluegreen-create-proxy.ts";
import {
  Site,
  Token,
  TRIGGER_SECRET,
  Writer,
  writerLayer,
} from "./fixtures/bluegreen-runtime-secrets/writer.ts";
import {
  BoundSecrets,
  CacheOne,
  CacheTwo,
  Site as SiteBluegreenSecrets,
} from "./fixtures/bluegreen-secrets.ts";
import {
  assertOrder,
  assertReplacement,
  assertStopped,
  makeScenario,
  requireValue,
} from "./fixtures/bluegreen-worker-test.ts";
import {
  assertAppGone,
  census,
  checks,
  deployWorker,
  assertCommitted,
} from "./fixtures/bluegreen.ts";
import { sanitizeExecFailure } from "./fixtures/exec-lease.ts";
import { makeReadinessControl, repairReadiness } from "./fixtures/http-readiness-control.ts";
import {
  assertReadinessCommit,
  readinessActor,
  readinessChecksPassing,
  readinessProxy,
  retires,
  type ReadinessEvent,
} from "./fixtures/idle-cadence-readiness.ts";
import {
  observeStops,
  type StopRequest,
  writeLegacyProtocol,
} from "./fixtures/legacy-protocol-writer.ts";
import MountedBlueGreen from "./fixtures/mounted-bluegreen.ts";
import {
  assertBarrierInventory,
  assertClean,
  assertConverged,
  assertSingleRunner,
  census as censusProcessDeath,
  deploy,
  evidencePaths,
  heldLeases,
  identity,
  machine,
  matchesBarrier,
  nowSeconds,
  observeLeaseExpiry,
  persistedRow,
  phases,
  processDeathFile,
  readWitness,
  writeEvidence,
  type Phase,
  type Witness,
} from "./fixtures/process-death.ts";
import {
  appName,
  candidateId,
  MachineServiceAutostop,
  metadata,
  protocolClient,
  reconcile,
  reply,
  withControlledClient,
} from "./fixtures/protocol-branches.ts";
import {
  Finalized,
  RunnerInterrupted,
  Witness as WitnessSignalOverlap,
  assertBoundary,
  assertConverged as assertConvergedSignalOverlap,
  assertInventory,
  assertReleased,
  assertSingleRunner as assertSingleRunnerSignalOverlap,
  boundary,
  cases,
  firstReturnedUncordon,
  matches,
  observeExpiry,
  observeLeases,
  pathsFor,
  readEvidence,
  signalOverlapFile,
  successful,
  type SignalCase,
} from "./fixtures/signal-overlap.ts";
import { delayedResourceWrite, ResourceRow } from "./fixtures/state-persistence.ts";
import { transportProxy, throughProxy, type TransportEvent } from "./fixtures/transport.ts";

// Blocks are independent (each case deploys its own stack). Fully parallel
// overloads Fly's API (slow App deletes, missed timing windows), so they run
// in six concurrent shards balanced by measured duration (~35 min each);
// blocks run one at a time within a shard, and process-wide cases keep
// `exclusive: true`.
/** URL param values may be `Redacted` (sent unwrapped); compare their plain text. */
const unredact = (value: string | Redacted.Redacted | undefined) =>
  Redacted.isRedacted(value) ? String(Redacted.value(value)) : value;

const decodeAutostop = Schema.decodeUnknownSync(MachineServiceAutostop);

describe.concurrent("BlueGreen", () => {
  describe.sequential("shard 1", () => {
    describe.sequential(
      "idle topology",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:machine",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        const file = "test/Fly/BlueGreen.test.ts";
        const { test } = Test.make({ providers: Fly.providers() });
        type IdleMode = "stop" | "suspend";

        const topology = (
          mode: IdleMode,
          count: number,
          floor = 0,
          mixed = false,
        ): Partial<Omit<Extract<MachineProps, { image: string }>, "app">> => ({
          count,
          deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
          services: [
            {
              protocol: "tcp",
              internalPort: 80,
              autostop: mode,
              autostart: true,
              minMachinesRunning: floor,
              checks: [checks.ready],
            },
            ...(mixed ? [{ protocol: "tcp", internalPort: 81, autostop: "off" as const }] : []),
          ],
        });

        const waitState = (appName: string, machineId: string, state: string) =>
          machines.getMachine({ app_name: appName, machine_id: machineId }).pipe(
            Effect.repeat({
              schedule: Schedule.spaced("2 seconds"),
              times: 30,
              until: (machine) => machine.state === state,
            }),
            Effect.tap((machine) => Effect.sync(() => expect(machine.state).toBe(state))),
            Effect.timeout("90 seconds"),
          );

        // Native stop/suspend is setup only; automatic return-to-idle has its own test.
        const idleAll = (appName: string, ids: string[], mode: IdleMode) =>
          Effect.forEach(ids, (id) =>
            Effect.gen(function* () {
              const target = { app_name: appName, machine_id: id };
              const current = yield* machines.getMachine(target);
              if (current.state !== "started") {
                yield* machines.startMachine(target);
                yield* waitState(appName, id, "started");
              }
              if (mode === "stop") {
                yield* machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" });
              } else {
                yield* machines.suspendMachine(target);
              }
              yield* waitState(appName, id, mode === "stop" ? "stopped" : "suspended");
            }),
          );

        // Expected slots are literal scenario inputs, never computed by readinessRoles.
        const assertSlots = (live: machines.Machine[], runningSlots: readonly number[]) => {
          const ordered = [...live].sort(
            (left, right) =>
              Number(left.config?.metadata?.["alchemy.replica"]) -
              Number(right.config?.metadata?.["alchemy.replica"]),
          );
          expect(ordered.map((machine) => machine.config?.metadata?.["alchemy.replica"])).toEqual(
            Array.from({ length: live.length }, (_, index) => String(index)),
          );
          expect(
            ordered
              .filter((machine) => machine.config?.metadata?.["alchemy.readiness-role"] === "run")
              .map((machine) => Number(machine.config?.metadata?.["alchemy.replica"])),
          ).toEqual([...runningSlots]);
          const roles = Array.from({ length: live.length }, (_, slot) =>
            runningSlots.includes(slot) ? "run" : "idle",
          ).join(",");
          for (const [slot, machine] of ordered.entries()) {
            expect(machine.config?.metadata?.["alchemy.readiness-role"]).toBe(
              runningSlots.includes(slot) ? "run" : "idle",
            );
            expect(machine.config?.metadata?.["alchemy.readiness-roles"]).toBe(roles);
          }
          return ordered;
        };

        const assertTopology = (
          appName: string,
          ids: string[],
          mode: IdleMode,
          runningSlots: readonly number[],
          floor = 0,
          mixed = false,
        ) =>
          Effect.gen(function* () {
            const live = yield* assertCommitted(appName, ids);
            const ordered = assertSlots(live, runningSlots);
            expect(
              new Set(live.map((machine) => machine.config?.metadata?.["alchemy.generation"])).size,
            ).toBe(1);
            for (const [slot, machine] of ordered.entries()) {
              expect(machine.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
              expect(machine.cordoned).toBe(false);
              expect(machine.config?.services).toHaveLength(mixed ? 2 : 1);
              expect(autostopMode(machine.config?.services?.[0]?.autostop)).toBe(mode);
              expect(machine.config?.services?.[0]?.autostart).toBe(true);
              expect(machine.config?.services?.[0]?.min_machines_running).toBe(floor);
              if (mixed) {
                expect(autostopMode(machine.config?.services?.[1]?.autostop)).toBe("off");
              }
              if (!runningSlots.includes(slot)) {
                expect(["created", "stopped", "suspended"]).toContain(machine.state);
              } else {
                expect(mixed ? ["started"] : ["started", "stopped", "suspended"]).toContain(
                  machine.state,
                );
                expect(machine.instance_id).toBeDefined();
                expect(machine.config?.metadata?.["alchemy.checked-instance"]).toBe(
                  machine.instance_id,
                );
              }
            }
            return ordered;
          });

        const checkedDeployment = (
          actor: Test.ScratchStack,
          proxy: { events: TransportEvent[]; readiness: ReadinessEvent[] },
          version: string,
          mode: IdleMode,
          count: number,
          runningSlots: readonly number[],
          priorIds: readonly string[] = [],
          floor = 0,
          mixed = false,
        ) =>
          Effect.gen(function* () {
            const begin = proxy.events.length;
            const proofBegin = proxy.readiness.length;
            const output = yield* deployWorker(
              actor,
              version,
              topology(mode, count, floor, mixed),
            ).pipe(Effect.scoped);
            const live = yield* assertTopology(
              output.appName,
              output.machineIds,
              mode,
              runningSlots,
              floor,
              mixed,
            );
            assertReadinessCommit(
              proxy.readiness.slice(proofBegin),
              priorIds,
              live,
              runningSlots,
              ["ready", "servicecheck-00-http-80"],
              !mixed,
            );
            expect(
              proxy.events
                .slice(begin)
                .some(
                  (event) =>
                    event.stage === "request" &&
                    priorIds.includes(event.machineId!) &&
                    event.path.endsWith("/start"),
                ),
            ).toBe(false);
            return output;
          });

        // Each case deploys its own stack; these are the file's longest cases
        // (3-8 min of real idle waits), so run them side by side.
        describe.concurrent("live idle topology and automatic cadence", () => {
          for (const mode of ["stop", "suspend"] as const) {
            test.provider(
              `S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
              (stack) =>
                Effect.gen(function* () {
                  yield* stack.destroy();
                  const site = yield* stack.deploy(Fly.App("Site"));
                  try {
                    const proxy = yield* readinessProxy();
                    const actor = yield* readinessActor(
                      stack,
                      `S05 S11 F13 ${mode} real all-idle 1 to 3 to 1 scaling preserves representative-only readiness`,
                      file,
                      proxy,
                    );
                    const first = yield* checkedDeployment(actor, proxy, "one", mode, 1, [0]);
                    yield* idleAll(site.appName, first.machineIds, mode);
                    const up = yield* checkedDeployment(
                      actor,
                      proxy,
                      "two",
                      mode,
                      3,
                      [0],
                      first.machineIds,
                    );
                    expect(up.machineIds).toHaveLength(3);
                    expect(up.machineIds.every((id) => !first.machineIds.includes(id))).toBe(true);
                    yield* idleAll(site.appName, up.machineIds, mode);
                    const down = yield* checkedDeployment(
                      actor,
                      proxy,
                      "three",
                      mode,
                      1,
                      [0],
                      up.machineIds,
                    );
                    expect(down.machineIds).toHaveLength(1);
                    expect(up.machineIds).not.toContain(down.machineId);
                    yield* idleAll(site.appName, down.machineIds, mode);
                    const unchanged = yield* deployWorker(actor, "three", topology(mode, 1));
                    expect(unchanged.machineIds).toEqual(down.machineIds);
                    expect((yield* census(site.appName))[0]!.state).toBe(
                      mode === "stop" ? "stopped" : "suspended",
                    );
                  } finally {
                    yield* stack.destroy();
                    yield* assertAppGone(site.appName);
                  }
                }).pipe(Effect.scoped),
              { timeout: 900_000 },
            );

            test.provider(
              `S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
              (stack) =>
                Effect.gen(function* () {
                  yield* stack.destroy();
                  const site = yield* stack.deploy(Fly.App("Site"));
                  try {
                    const proxy = yield* readinessProxy();
                    const actor = yield* readinessActor(
                      stack,
                      `S05 S11 F13 ${mode} live floor 0 to 2 to 0 and mixed off-plus-idle service topology`,
                      file,
                      proxy,
                    );
                    const first = yield* checkedDeployment(actor, proxy, "one", mode, 3, [0]);
                    yield* idleAll(site.appName, first.machineIds, mode);
                    const raised = yield* checkedDeployment(
                      actor,
                      proxy,
                      "one",
                      mode,
                      3,
                      [0, 1],
                      first.machineIds,
                      2,
                    );
                    yield* idleAll(site.appName, raised.machineIds, mode);
                    const lowered = yield* checkedDeployment(
                      actor,
                      proxy,
                      "one",
                      mode,
                      3,
                      [0],
                      raised.machineIds,
                    );
                    yield* idleAll(site.appName, lowered.machineIds, mode);
                    const slots = assertSlots(yield* census(site.appName), [0]);
                    yield* machines.startMachine({
                      app_name: site.appName,
                      machine_id: slots[2]!.id!,
                    });
                    yield* waitState(site.appName, slots[2]!.id!, "started");
                    const mixedOld = (yield* census(site.appName)).sort(
                      (a, b) =>
                        Number(a.config?.metadata?.["alchemy.replica"]) -
                        Number(b.config?.metadata?.["alchemy.replica"]),
                    );
                    expect(mixedOld.map((machine) => machine.state)).toEqual([
                      mode === "stop" ? "stopped" : "suspended",
                      mode === "stop" ? "stopped" : "suspended",
                      "started",
                    ]);
                    const retained = yield* checkedDeployment(
                      actor,
                      proxy,
                      "retained",
                      mode,
                      3,
                      [2],
                      lowered.machineIds,
                    );
                    yield* idleAll(site.appName, retained.machineIds, mode);
                    const retainedSlots = assertSlots(yield* census(site.appName), [2]);
                    yield* machines.startMachine({
                      app_name: site.appName,
                      machine_id: retainedSlots[2]!.id!,
                    });
                    yield* waitState(site.appName, retainedSlots[2]!.id!, "started");
                    expect(
                      assertSlots(yield* census(site.appName), [2]).map((machine) => machine.state),
                    ).toEqual([
                      mode === "stop" ? "stopped" : "suspended",
                      mode === "stop" ? "stopped" : "suspended",
                      "started",
                    ]);
                    const retainedFloor = yield* checkedDeployment(
                      actor,
                      proxy,
                      "retained",
                      mode,
                      3,
                      [0, 2],
                      retained.machineIds,
                      2,
                    );
                    yield* idleAll(site.appName, retainedFloor.machineIds, mode);
                    const mixed = yield* checkedDeployment(
                      actor,
                      proxy,
                      "one",
                      mode,
                      3,
                      [0, 1, 2],
                      retainedFloor.machineIds,
                      0,
                      true,
                    );
                    yield* idleAll(site.appName, mixed.machineIds, mode);
                    yield* checkedDeployment(actor, proxy, "one", mode, 3, [0], mixed.machineIds);
                  } finally {
                    yield* stack.destroy();
                    yield* assertAppGone(site.appName);
                  }
                }).pipe(Effect.scoped),
              { timeout: 1_200_000 },
            );

            for (const recoveredCount of [3, 1, 4]) {
              test.provider(
                `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
                (stack) =>
                  Effect.gen(function* () {
                    yield* stack.destroy();
                    const site = yield* stack.deploy(Fly.App("Site"));
                    try {
                      const proxy = yield* readinessProxy();
                      const initial = yield* readinessActor(
                        stack,
                        `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
                        file,
                        proxy,
                      );
                      const first = yield* checkedDeployment(initial, proxy, "one", mode, 3, [0]);
                      yield* idleAll(site.appName, first.machineIds, mode);
                      const source = (yield* census(site.appName))[0]!;
                      const metadata = source.config!.metadata!;
                      const attemptBegin = proxy.events.length;
                      yield* Effect.gen(function* () {
                        try {
                          const gate = yield* Deferred.make<string>();
                          const actor = yield* readinessActor(
                            stack,
                            `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
                            file,
                            proxy,
                            (event) =>
                              event.method === "POST" &&
                              event.path ===
                                `/v1/apps/${site.appName}/machines/${event.machineId}` &&
                              event.metadata?.["alchemy.replica"] === "1" &&
                              event.metadata["alchemy.fqn"] === metadata["alchemy.fqn"] &&
                              event.metadata["alchemy.generation"] !==
                                metadata["alchemy.generation"] &&
                              event.metadata["alchemy.readiness-roles"] === "run,run,idle" &&
                              event.metadata["alchemy.idle-policy-restored"] === "true" &&
                              event.phase === "promoting"
                                ? Deferred.succeed(gate, event.machineId!).pipe(
                                    Effect.andThen(Effect.never),
                                  )
                                : Effect.void,
                          );
                          const attempt = yield* deployWorker(
                            actor,
                            "two",
                            topology(mode, 3, 2),
                          ).pipe(Effect.scoped, Effect.forkScoped);
                          const barrierMachine = yield* Deferred.await(gate).pipe(
                            Effect.timeout("180 seconds"),
                          );
                          const pending = yield* census(site.appName);
                          expect(
                            pending.filter((machine) => first.machineIds.includes(machine.id!)),
                          ).toHaveLength(3);
                          expect(
                            proxy.events.some((event) => retires(event, first.machineIds)),
                          ).toBe(false);
                          const candidates = yield* Effect.forEach(
                            pending.filter((machine) => !first.machineIds.includes(machine.id!)),
                            (machine) =>
                              machines.getMachine({
                                app_name: site.appName,
                                machine_id: machine.id!,
                              }),
                          );
                          expect(candidates).toHaveLength(3);
                          expect(
                            candidates.every(
                              (machine) =>
                                machine.config?.metadata?.["alchemy.phase"] === "promoting",
                            ),
                          ).toBe(true);
                          const pendingSlots = assertSlots(candidates, [0, 1]);
                          expect(
                            pendingSlots[0]!.config?.metadata?.["alchemy.idle-policy-restored"],
                          ).toBe("true");
                          expect(
                            pendingSlots[1]!.config?.metadata?.["alchemy.idle-policy-restored"],
                          ).toBe("false");
                          expect(
                            proxy.readiness.some(
                              (event) =>
                                event.stage === "forwarded" &&
                                event.method === "GET" &&
                                event.machineId === pendingSlots[0]!.id &&
                                event.instanceId === pendingSlots[0]!.instance_id &&
                                event.metadata?.["alchemy.idle-policy-restored"] === "true" &&
                                event.phase === "promoting" &&
                                event.state === "started" &&
                                readinessChecksPassing(event.checks, [
                                  "ready",
                                  "servicecheck-00-http-80",
                                ]),
                            ),
                          ).toBe(true);
                          expect(barrierMachine).toBe(pendingSlots[1]!.id);
                          expect(
                            autostopMode(pendingSlots[0]!.config?.services?.[0]?.autostop),
                          ).toBe(mode);
                          expect(
                            autostopMode(pendingSlots[1]!.config?.services?.[0]?.autostop),
                          ).toBe("off");
                          expect(
                            proxy.events
                              .slice(attemptBegin)
                              .some(
                                (event) =>
                                  event.method === "POST" &&
                                  event.path.endsWith(`/machines/${barrierMachine}`) &&
                                  event.phase === "promoting",
                              ),
                          ).toBe(false);
                          const read = yield* observeReplicaSet({
                            appName: site.appName,
                            id: "Worker",
                            type: "Fly.Machine",
                            fqn: metadata["alchemy.fqn"]!,
                            resourceInstanceId: metadata["alchemy.instance"]!,
                            baseName: first.baseName,
                            machineIds: first.machineIds,
                          });
                          expect(read?.machineIds).toEqual(first.machineIds);
                          expect(read?.count).toBe(3);
                          expect(read?.rolloutPending).toBe(true);
                          const interruption = yield* Fiber.interrupt(attempt).pipe(
                            Effect.forkScoped,
                          );
                          yield* Effect.yieldNow;
                          yield* Effect.sync(() => {
                            proxy.clear();
                            proxy.release();
                          });
                          yield* Fiber.join(interruption).pipe(Effect.timeout("180 seconds"));
                          expect(Exit.hasInterrupts(yield* Fiber.await(attempt))).toBe(true);
                          expect(
                            proxy.events
                              .slice(attemptBegin)
                              .some(
                                (event) =>
                                  first.machineIds.includes(event.machineId!) &&
                                  event.path.endsWith("/start"),
                              ),
                          ).toBe(false);
                          expect(
                            proxy.readiness.some(
                              (event) =>
                                event.stage === "request" &&
                                event.path.endsWith("/metadata") &&
                                event.phase === "active" &&
                                candidates.some((machine) => machine.id === event.machineId),
                            ),
                          ).toBe(false);
                          expect(
                            proxy.events.some((event) => retires(event, first.machineIds)),
                          ).toBe(false);
                          const resumed = yield* readinessActor(
                            stack,
                            `S05 S11 F08 F13 ${mode} interrupted partial idle restoration recovers count 3 to ${recoveredCount}`,
                            file,
                            proxy,
                          );
                          const begin = proxy.readiness.length;
                          const recovered = yield* deployWorker(
                            resumed,
                            "two",
                            topology(mode, recoveredCount, 2),
                          ).pipe(Effect.scoped);
                          expect(recovered.machineIds).toHaveLength(recoveredCount);
                          if (recoveredCount === 3) {
                            expect([...recovered.machineIds].sort()).toEqual(
                              candidates.map((machine) => machine.id).sort(),
                            );
                          } else {
                            expect(
                              recovered.machineIds.every(
                                (id) => !pending.some((machine) => machine.id === id),
                              ),
                            ).toBe(true);
                          }
                          const runningSlots = recoveredCount === 1 ? [0] : [0, 1];
                          const live = yield* assertTopology(
                            site.appName,
                            recovered.machineIds,
                            mode,
                            runningSlots,
                            2,
                          );
                          assertReadinessCommit(
                            proxy.readiness.slice(begin),
                            pending
                              .map((machine) => machine.id!)
                              .filter((id) => !recovered.machineIds.includes(id)),
                            live,
                            runningSlots,
                            ["ready", "servicecheck-00-http-80"],
                            true,
                          );
                          const unchanged = yield* deployWorker(
                            resumed,
                            "two",
                            topology(mode, recoveredCount, 2),
                          ).pipe(Effect.scoped);
                          expect(unchanged.machineIds).toEqual(recovered.machineIds);
                          const committed = yield* observeReplicaSet({
                            appName: site.appName,
                            id: "Worker",
                            type: "Fly.Machine",
                            fqn: metadata["alchemy.fqn"]!,
                            resourceInstanceId: metadata["alchemy.instance"]!,
                            machineIds: recovered.machineIds,
                          });
                          expect(committed?.rolloutPending).toBe(false);
                          expect(committed?.machineIds).toEqual(recovered.machineIds);
                        } finally {
                          yield* Effect.sync(() => {
                            proxy.clear();
                            proxy.release();
                          });
                        }
                      }).pipe(Effect.scoped);
                    } finally {
                      yield* stack.destroy();
                      yield* assertAppGone(site.appName);
                    }
                  }).pipe(Effect.scoped),
                { timeout: 900_000 },
              );
            }

            test.provider(
              `S11 public traffic autostarts ${mode} and Fly automatically returns it to idle within 15 minutes`,
              (stack) =>
                Effect.gen(function* () {
                  yield* stack.destroy();
                  const site = yield* stack.deploy(Fly.App("Site"));
                  try {
                    const output = yield* stack.deploy(
                      Effect.gen(function* () {
                        const app = yield* Fly.App("Site");
                        yield* Fly.IpAssignment("Public", { app, type: "shared_v4" });
                        return yield* Fly.Machine("Worker", {
                          app,
                          image: "nginx:alpine",
                          deploy: { strategy: "bluegreen", healthTimeout: "60 seconds" },
                          shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
                          services: [
                            {
                              protocol: "tcp",
                              internalPort: 80,
                              autostop: mode,
                              autostart: true,
                              minMachinesRunning: 0,
                              ports: [{ port: 80, handlers: ["http"] }],
                              checks: [checks.ready],
                            },
                          ],
                        });
                      }),
                    );
                    yield* idleAll(site.appName, output.machineIds, mode);
                    const client = yield* HttpClient.HttpClient;
                    yield* Effect.gen(function* () {
                      const response = yield* client.get(`http://${site.appName}.fly.dev`);
                      const body = yield* response.text;
                      if (response.status !== 200 || !body.includes("Welcome to nginx")) {
                        return yield* Effect.fail(
                          new Error(`Public Fly route is not ready: ${response.status}`),
                        );
                      }
                      expect(response.status).toBe(200);
                      expect(body).toContain("Welcome to nginx");
                    }).pipe(
                      Effect.retry({ schedule: Schedule.spaced("3 seconds"), times: 20 }),
                      Effect.timeout("90 seconds"),
                    );
                    yield* waitState(site.appName, output.machineId, "started");
                    const started = yield* Clock.currentTimeMillis;
                    const idleState = mode === "stop" ? "stopped" : "suspended";
                    // Only native GETs follow the last public request; no stop, suspend, or update substitutes.
                    const idle = yield* machines
                      .getMachine({ app_name: site.appName, machine_id: output.machineId })
                      .pipe(
                        Effect.tap((machine) =>
                          Effect.logInfo("Automatic Fly idle observation", {
                            mode,
                            state: machine.state,
                            instanceId: machine.instance_id,
                          }),
                        ),
                        Effect.repeat({
                          schedule: Schedule.spaced("10 seconds"),
                          times: 90,
                          until: (machine) => machine.state === idleState,
                        }),
                        Effect.timeout("15 minutes"),
                      );
                    expect(idle.state).toBe(idleState);
                    expect(idle.cordoned).toBe(false);
                    expect(autostopMode(idle.config?.services?.[0]?.autostop)).toBe(mode);
                    expect(idle.config?.metadata?.["alchemy.idle-policy-restored"]).toBe("true");
                    yield* Effect.logInfo("Automatic return to idle observed", {
                      mode,
                      elapsedMs: (yield* Clock.currentTimeMillis) - started,
                    });
                    expect((yield* census(site.appName)).map((machine) => machine.id)).toEqual(
                      output.machineIds,
                    );
                  } finally {
                    yield* stack.destroy();
                    yield* assertAppGone(site.appName);
                  }
                }),
              { tags: ["provider:fly:ipassignment"], timeout: 1_200_000 },
            );
          }
        });
      },
    );

    describe.sequential(
      "idle capacity",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:machine",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        let endpoint: string | undefined;
        const { test } = Test.make({ providers: throughProxy(() => endpoint) });

        describe.sequential("idle topology", () => {
          for (const mode of ["stop", "suspend"] as const) {
            for (const allIdle of [false, true]) {
              test.provider(
                `S05 S11 ${mode} ${allIdle ? "all-idle" : "mixed"} replacement keeps an idle nonrepresentative`,
                (stack) =>
                  Effect.gen(function* () {
                    yield* stack.destroy();
                    const props = {
                      count: 2,
                      services: [
                        {
                          protocol: "tcp",
                          internalPort: 80,
                          autostop: mode,
                          autostart: true,
                          minMachinesRunning: 0,
                          ports: [{ port: 80, handlers: ["http"] }],
                          checks: [checks.ready],
                        },
                      ],
                    };
                    const initial = yield* deployWorker(stack, "one", {
                      ...props,
                      deploy: { strategy: "rolling" },
                    });
                    for (const id of allIdle ? initial.machineIds : initial.machineIds.slice(1)) {
                      const target = { app_name: initial.appName, machine_id: id };
                      yield* mode === "stop"
                        ? machines.stopMachine({ ...target, signal: "SIGQUIT", timeout: "5s" })
                        : machines.suspendMachine(target);
                      const observed = yield* machines.getMachine(target).pipe(
                        Effect.repeat({
                          schedule: Schedule.spaced("2 seconds"),
                          times: 8,
                          until: (machine) =>
                            machine.state === (mode === "stop" ? "stopped" : "suspended"),
                        }),
                      );
                      expect(observed.state).toBe(mode === "stop" ? "stopped" : "suspended");
                    }
                    const proxy = yield* transportProxy();
                    yield* Effect.sync(() => {
                      endpoint = proxy.url;
                    });
                    const next = yield* deployWorker(stack, "two", props);
                    expect(next.machineIds).toHaveLength(2);
                    const live = yield* census(initial.appName);
                    expect(live.map((machine) => machine.id).sort()).toEqual(
                      [...next.machineIds].sort(),
                    );
                    expect(
                      live.every((machine) => {
                        const autostop = machine.config?.services?.[0]?.autostop;
                        return (autostop === true ? "stop" : autostop) === mode;
                      }),
                    ).toBe(true);
                    expect(
                      live.filter((machine) => machine.state === "started").length,
                    ).toBeLessThanOrEqual(1);
                    expect(
                      live.some((machine) => ["stopped", "suspended"].includes(machine.state!)),
                    ).toBe(true);
                    expect(
                      proxy.events.some(
                        (event) =>
                          initial.machineIds.includes(event.machineId!) &&
                          event.path.endsWith("/start"),
                      ),
                    ).toBe(false);
                    const same = yield* deployWorker(stack, "two", props);
                    expect(same.machineIds).toEqual(next.machineIds);
                    yield* Effect.sync(() => {
                      endpoint = undefined;
                    });
                    yield* stack.destroy();
                    yield* assertAppGone(initial.appName);
                  }).pipe(
                    Effect.scoped,
                    Effect.ensuring(
                      Effect.sync(() => {
                        endpoint = undefined;
                      }),
                    ),
                  ),
                { timeout: 300_000 },
              );
            }
          }
        });
      },
    );

    describe.sequential(
      "post-promotion health",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:ipassignment",
          "provider:fly:machine",
          "provider:fly:secret",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        let endpoint: string | undefined;
        const { test } = Test.make({ providers: throughProxy(() => endpoint) });

        describe.sequential("post-promotion health", () => {
          for (const changed of [false, true]) {
            test.provider(
              `F09 a real post-uncordon health flip preserves both generations then recovers ${changed ? "changed" : "same"} desired state`,
              (stack) =>
                Effect.gen(function* () {
                  yield* stack.destroy();
                  const readiness = yield* makeReadinessControl();
                  const initial = yield* readiness.deployWorker(stack, "one");
                  const proxy = yield* transportProxy();
                  yield* Effect.sync(() => {
                    endpoint = proxy.url;
                    proxy.arm({
                      match: (event) => event.path.endsWith("/uncordon"),
                      action: "hold-response",
                      remaining: 1,
                    });
                  });
                  const update = yield* readiness
                    .deployWorker(stack, "two")
                    .pipe(Effect.result, Effect.forkScoped);
                  const promoted = yield* proxy.wait(
                    (event) =>
                      event.stage === "held" &&
                      event.path.endsWith("/uncordon") &&
                      event.status! < 300,
                  );
                  expect(promoted.machineId).toBeDefined();
                  yield* readiness.turnOff(initial.appName, promoted.machineId!);
                  yield* Effect.sync(proxy.release);
                  const result = yield* Fiber.join(update).pipe(Effect.timeout("60 seconds"));
                  expect(Result.isFailure(result)).toBe(true);
                  if (Result.isFailure(result))
                    expect(result.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
                  const live = yield* census(initial.appName);
                  expect(live.map((machine) => machine.id).sort()).toEqual(
                    [initial.machineId, promoted.machineId!].sort(),
                  );
                  expect(live.every((machine) => machine.cordoned === false)).toBe(true);
                  const pending = live.find((machine) => machine.id === promoted.machineId)!;
                  const metadata = pending.config!.metadata!;
                  expect(metadata["alchemy.phase"]).toBe("validating");
                  expect(metadata["alchemy.checked-instance"]).toBeUndefined();
                  const read = yield* observeReplicaSet({
                    appName: initial.appName,
                    id: "Worker",
                    type: "Fly.Machine",
                    fqn: metadata["alchemy.fqn"]!,
                    resourceInstanceId: metadata["alchemy.instance"]!,
                    machineIds: initial.machineIds,
                  });
                  expect(read?.machineIds).toEqual(initial.machineIds);
                  expect(read?.rolloutPending).toBe(true);
                  expect(
                    proxy.events.some(
                      (event) => event.method === "DELETE" && /\/machines\/[^/]+$/.test(event.path),
                    ),
                  ).toBe(false);
                  if (!changed) {
                    yield* repairReadiness(initial.appName, promoted.machineId!);
                  }
                  const recovered = yield* readiness.deployWorker(stack, changed ? "three" : "two");
                  if (changed) expect(recovered.machineId).not.toBe(promoted.machineId);
                  else expect(recovered.machineId).toBe(promoted.machineId);
                  yield* assertCommitted(initial.appName, recovered.machineIds);
                  yield* Effect.sync(() => {
                    endpoint = undefined;
                  });
                  yield* stack.destroy();
                  yield* assertAppGone(initial.appName);
                }).pipe(
                  Effect.scoped,
                  Effect.ensuring(
                    Effect.sync(() => {
                      endpoint = undefined;
                    }),
                  ),
                ),
              { timeout: 300_000 },
            );
          }
        });
      },
    );

    describe.sequential(
      "ownership",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:machine",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        const { test } = Test.make({ providers: Fly.providers() });

        test.provider(
          "S09 foreign routed Machine and sibling survive rollout and stale-instance deletion",
          (stack) =>
            Effect.gen(function* () {
              yield* stack.destroy();
              const deploy = (version: string) =>
                stack.deploy(
                  Effect.gen(function* () {
                    const app = yield* Fly.App("Site");
                    const sibling = yield* Fly.Machine("Sibling", {
                      app,
                      image: "nginx:alpine",
                      env: { VERSION: "sibling" },
                    });
                    const worker = yield* Fly.Machine("Worker", {
                      app,
                      image: "nginx:alpine",
                      env: { VERSION: version },
                      checks,
                      deploy: { strategy: "bluegreen", healthTimeout: "20 seconds" },
                      shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
                    });
                    return { sibling, worker };
                  }),
                );
              const initial = yield* deploy("one");
              const foreign = yield* machines.createMachine({
                app_name: initial.worker.appName,
                name: "foreign-routed",
                region: "iad",
                config: {
                  image: "nginx:alpine",
                  env: { VERSION: "foreign" },
                  services: [
                    {
                      protocol: "tcp",
                      internal_port: 80,
                      ports: [{ port: 80, handlers: ["http"] }],
                    },
                  ],
                },
              });
              const removeForeign = machines
                .deleteMachine({
                  app_name: initial.worker.appName,
                  machine_id: foreign.id!,
                  force: true,
                })
                .pipe(Effect.catchTag("NotFound", () => Effect.void));
              yield* Effect.gen(function* () {
                const next = yield* deploy("two");
                expect(next.worker.machineId).not.toBe(initial.worker.machineId);
                expect(next.sibling.machineId).toBe(initial.sibling.machineId);
                const before = yield* census(initial.worker.appName);
                expect(before.map((machine) => machine.id).sort()).toEqual(
                  [next.worker.machineId, next.sibling.machineId, foreign.id!].sort(),
                );
                expect(before.find((machine) => machine.id === foreign.id)?.cordoned).toBe(false);
                const owned = before.find((machine) => machine.id === next.worker.machineId)!;
                yield* deleteReplicaSet({
                  appName: next.worker.appName,
                  id: "Worker",
                  type: "Fly.Machine",
                  fqn: owned.config!.metadata!["alchemy.fqn"]!,
                  resourceInstanceId: "stale-instance-must-not-delete-successor",
                  machineIds: next.worker.machineIds,
                  volumeIds: [],
                });
                expect(
                  (yield* census(initial.worker.appName)).map((machine) => machine.id).sort(),
                ).toEqual(before.map((machine) => machine.id).sort());
              }).pipe(Effect.ensuring(removeForeign.pipe(Effect.orDie)));
              yield* stack.destroy();
              yield* assertAppGone(initial.worker.appName);
            }),
          { timeout: 300_000 },
        );

        test.provider(
          "S09 F11 stale output and a same-logical-ID foreign FQN cannot authorize successor deletion",
          (stack) =>
            Effect.gen(function* () {
              yield* stack.destroy();
              const deploy = (name: string, version: string) =>
                stack.deploy(
                  Effect.gen(function* () {
                    const app = yield* Fly.App("Site");
                    return yield* Fly.Machine("Worker", {
                      app,
                      name,
                      image: "nginx:alpine",
                      checks,
                      env: { VERSION: version },
                      deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
                      shutdown: { signal: "SIGQUIT", timeout: "5 seconds" },
                    });
                  }),
                );
              const initial = yield* deploy("stale-original", "one");
              const old = yield* machines.getMachine({
                app_name: initial.appName,
                machine_id: initial.machineId,
              });
              const oldMetadata = old.config!.metadata!;
              const collision = yield* machines.createMachine({
                app_name: initial.appName,
                name: "fqn-collision",
                region: "iad",
                config: {
                  ...old.config,
                  metadata: {
                    ...oldMetadata,
                    "alchemy.fqn": `${oldMetadata["alchemy.fqn"]}/OtherScope`,
                  },
                },
              });
              yield* Effect.gen(function* () {
                const next = yield* deploy("stale-original", "two");
                const refreshed = yield* observeReplicaSet({
                  appName: initial.appName,
                  id: "Worker",
                  type: "Fly.Machine",
                  fqn: oldMetadata["alchemy.fqn"]!,
                  resourceInstanceId: oldMetadata["alchemy.instance"]!,
                  baseName: initial.baseName,
                  machineIds: [...initial.machineIds, collision.id!],
                });
                expect(refreshed?.machineIds).toEqual(next.machineIds);
                expect(refreshed?.rolloutPending).toBe(false);
                const successor = yield* deploy("stale-successor", "three");
                const current = yield* machines.getMachine({
                  app_name: initial.appName,
                  machine_id: successor.machineId,
                });
                expect(current.config?.metadata?.["alchemy.instance"]).not.toBe(
                  oldMetadata["alchemy.instance"],
                );
                // Even an ID cache containing the new Machine grants no old-lineage authority.
                yield* deleteReplicaSet({
                  appName: initial.appName,
                  id: "Worker",
                  type: "Fly.Machine",
                  fqn: oldMetadata["alchemy.fqn"]!,
                  resourceInstanceId: oldMetadata["alchemy.instance"]!,
                  machineIds: [
                    ...initial.machineIds,
                    ...next.machineIds,
                    ...successor.machineIds,
                    collision.id!,
                  ],
                  volumeIds: [],
                });
                const live = yield* census(initial.appName);
                expect(live.map((machine) => machine.id).sort()).toEqual(
                  [successor.machineId, collision.id!].sort(),
                );
                const foreign = live.find((machine) => machine.id === collision.id)!;
                expect(foreign.config?.metadata?.["alchemy.fqn"]).toBe(
                  `${oldMetadata["alchemy.fqn"]}/OtherScope`,
                );
                expect(foreign.cordoned).toBe(false);
              }).pipe(
                Effect.ensuring(
                  machines
                    .deleteMachine({
                      app_name: initial.appName,
                      machine_id: collision.id!,
                      force: true,
                    })
                    .pipe(
                      Effect.catchTag("NotFound", () => Effect.void),
                      Effect.orDie,
                    ),
                ),
              );
              yield* stack.destroy();
              yield* assertAppGone(initial.appName);
            }),
          { timeout: 600_000 },
        );
      },
    );

    describe.sequential(
      "deployment",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:machine",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        const { test } = Test.make({ providers: Fly.providers() });

        test.provider(
          "S01 bluegreen worker checks, promotion, replacement, and graceful teardown",
          (stack) =>
            Effect.gen(function* () {
              yield* stack.destroy();
              const deploy = (version: string, path = "/") =>
                stack.deploy(
                  Effect.gen(function* () {
                    const app = yield* Fly.App("Site");
                    return yield* Fly.Machine("Worker", {
                      app,
                      name: "bluegreen-worker-long-name-base",
                      image: "nginx:alpine",
                      env: { VERSION: version },
                      deploy: { strategy: "bluegreen", healthTimeout: "30 seconds" },
                      shutdown: { signal: "SIGQUIT", timeout: "10 seconds" },
                      checks: {
                        ready: { type: "http", port: 80, path, interval: "2s", timeout: "1s" },
                      },
                    });
                  }),
                );
              const first = yield* deploy("one");
              const committed = yield* machines.getMachine({
                app_name: first.appName,
                machine_id: first.machineId,
              });
              // Active commit metadata can reset reports after the provider's readiness validation.
              const machine = yield* waitHealthy(first.appName, committed, 30_000);
              expect(machine.instance_id).toBe(committed.instance_id);
              yield* Effect.logInfo("Observed promoted worker", {
                checks: machine.checks?.map(({ name, status }) => ({ name, status })),
                configured: machine.config?.checks,
                metadata: machine.config?.metadata,
              });
              expect(machine.cordoned).toBe(false);
              expect(
                machine.checks?.some(
                  (check) => check.name === "ready" && check.status === "passing",
                ),
              ).toBe(true);
              expect(machine.config?.metadata?.["alchemy.phase"]).toBe("active");
              expect(first.name.length).toBeLessThanOrEqual(30);
              const second = yield* deploy("two");
              expect(second.machineId).not.toBe(first.machineId);
              expect(second.baseName).toBe(first.baseName);
              const listed = yield* machines.listMachines({ app_name: first.appName });
              expect(
                listed
                  .filter((machine) => machine.state !== "destroyed")
                  .map((machine) => machine.id),
              ).toEqual([second.machineId]);
              yield* stack.destroy();
              yield* assertAppGone(first.appName);
            }),
          { timeout: 180_000 },
        );

        test.provider(
          "S04 unhealthy replacement preserves the old routed ID and cleans unpromoted candidates",
          (stack) =>
            Effect.gen(function* () {
              yield* stack.destroy();
              const initial = yield* deployWorker(stack, "one");
              const failed = yield* deployWorker(stack, "broken", {
                checks: { ready: { ...checks.ready, path: "/missing" } },
                deploy: { strategy: "bluegreen", healthTimeout: "8 seconds" },
              }).pipe(Effect.result);
              expect(Result.isFailure(failed)).toBe(true);
              if (Result.isFailure(failed))
                expect(failed.failure).toMatchObject({ _tag: "Fly.ReplicaChecksNotPassing" });
              const live = yield* census(initial.appName);
              expect(live.map((machine) => machine.id)).toEqual(initial.machineIds);
              expect(live[0]!.state).toBe("started");
              expect(live[0]!.cordoned).toBe(false);
              yield* stack.destroy();
              yield* assertAppGone(initial.appName);
            }),
          { timeout: 180_000 },
        );
      },
    );

    describe.sequential(
      "promotion faults",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:machine",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        let endpoint: string | undefined;
        const { test } = Test.make({ providers: throughProxy(() => endpoint) });

        test.provider(
          "F03 F08 partial promotion loses the first uncordon response without rolling back serving green",
          (stack) =>
            Effect.gen(function* () {
              yield* stack.destroy();
              const initial = yield* deployWorker(stack, "one", { count: 2 });
              const proxy = yield* transportProxy();
              const match = (event: { path: string }) => event.path.endsWith("/uncordon");
              yield* Effect.sync(() => {
                endpoint = proxy.url;
                proxy.arm({ match, action: "drop-response", remaining: 1 });
                proxy.arm({ match, action: "cut-request", remaining: Infinity });
              });
              try {
                const failed = yield* deployWorker(stack, "two", { count: 2 }).pipe(
                  Effect.timeout("120 seconds"),
                  Effect.result,
                );
                expect(Result.isFailure(failed)).toBe(true);
                const lost = proxy.events.find(
                  (event) => event.stage === "dropped" && event.path.endsWith("/uncordon"),
                );
                expect(lost?.status).toBeGreaterThanOrEqual(200);
                expect(lost?.status).toBeLessThan(300);
                const live = yield* census(initial.appName);
                expect(
                  initial.machineIds.every((id) =>
                    live.some((machine) => machine.id === id && machine.cordoned === false),
                  ),
                ).toBe(true);
                expect(live.find((machine) => machine.id === lost!.machineId)?.cordoned).toBe(
                  false,
                );
                const candidateIds = live
                  .filter((machine) => !initial.machineIds.includes(machine.id!))
                  .map((machine) => machine.id!);
                expect(candidateIds).toHaveLength(2);
                expect(
                  proxy.events.some(
                    (event) =>
                      event.method === "DELETE" &&
                      /\/machines\/[^/]+$/.test(event.path) &&
                      candidateIds.includes(event.machineId!),
                  ),
                ).toBe(false);
                yield* Effect.sync(proxy.clear);
                const recovered = yield* deployWorker(stack, "two", { count: 2 });
                expect([...recovered.machineIds].sort()).toEqual(candidateIds.sort());
                yield* assertCommitted(initial.appName, recovered.machineIds);
              } finally {
                yield* Effect.sync(() => {
                  endpoint = undefined;
                  proxy.clear();
                });
              }
              yield* stack.destroy();
              yield* assertAppGone(initial.appName);
            }).pipe(
              Effect.scoped,
              Effect.ensuring(
                Effect.sync(() => {
                  endpoint = undefined;
                }),
              ),
            ),
          { timeout: 300_000 },
        );
      },
    );

    describe.sequential(
      "transport",
      {
        tags: [
          "provider:fly",
          "provider:fly:app",
          "provider:fly:machine",
          "provider:fly:service",
          "live",
        ],
      },
      () => {
        let endpoint: string | undefined;
        const { test } = Test.make({ providers: throughProxy(() => endpoint) });

        test.provider(
          "F02 S07 harness forwards real Fly responses, loses completed responses and cuts connections",
          (stack) =>
            Effect.gen(function* () {
              yield* stack.destroy();
              const app = yield* stack.deploy(Fly.App("Site"));
              const proxy = yield* transportProxy();
              yield* Effect.sync(() => {
                endpoint = proxy.url;
              });
              try {
                const request = machines.getApp({ app_name: app.appName }).pipe(Retry.none);
                const forwarded = yield* request;
                expect(forwarded.name).toBe(app.appName);
                expect(
                  proxy.events.some((event) => event.stage === "forwarded" && event.status === 200),
                ).toBe(true);
                yield* Effect.sync(() =>
                  proxy.arm({
                    match: (event) => event.path.endsWith(`/apps/${app.appName}`),
                    action: "drop-response",
                    remaining: 1,
                  }),
                );
                const dropped = yield* request.pipe(Effect.result);
                // The HTTP runtime may transparently replay an idempotent GET after a socket reset.
                if (Result.isFailure(dropped)) expect(dropped.failure._tag).toBe("HttpClientError");
                else expect(dropped.success.name).toBe(app.appName);
                expect(
                  proxy.events.filter((event) => event.stage === "dropped" && event.status === 200),
                ).toHaveLength(1);
                yield* Effect.sync(() =>
                  proxy.arm({ match: () => true, action: "cut-request", remaining: Infinity }),
                );
                const cut = yield* request.pipe(Effect.result);
                expect(Result.isFailure(cut)).toBe(true);
                const cutEvent = proxy.events.find((event) => event.stage === "cut")!;
                expect(
                  proxy.events.some(
                    (event) => event.sequence === cutEvent.sequence && event.stage === "completed",
                  ),
                ).toBe(false);
                yield* Effect.sync(() => {
                  proxy.clear();
                  proxy.arm({ match: () => true, action: "hold-response", remaining: 1 });
                });
                const child = yield* request.pipe(Effect.forkScoped);
                const held = yield* proxy.wait((event) => event.stage === "held");
                expect(held.status).toBe(200);