sailfork / t3code-github / infra/ relay/ src/ environments/ ManagedEndpointProvider.test.ts

import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";

import { describe, expect, it } from "@effect/vitest";
import * as Alchemy from "alchemy";
import * as Cloudflare from "@alchemy.run/cloudflare";
import * as ZeroTrust from "@distilled.cloud/cloudflare/zero-trust";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Redacted from "effect/Redacted";

import * as RelayConfiguration from "../Config.ts";
import * as CloudflareApiPause from "./CloudflareApiPause.ts";
import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
import * as ManagedTunnelLimits from "./ManagedTunnelLimits.ts";

const config = RelayConfiguration.RelayConfiguration.of({
  relayIssuer: "https://relay.example.test",
  apns: {
    environment: "sandbox",
    teamId: "team-id",
    keyId: "key-id",
    privateKey: Redacted.make("private-key"),
    bundleId: "com.t3tools.t3code.dev",
  },
  apnsDeliveryJobSigningSecret: Redacted.make("job-secret"),
  clerkSecretKey: Redacted.make("clerk-secret"),
  clerkPublishableKey: "pk_test_test",
  clerkJwtAudience: "t3-code-relay",
  cloudMintPrivateKey: Redacted.make("cloud-private-key"),
  cloudMintPublicKey: "cloud-public-key",
  managedEndpointBaseDomain: "t3code.test",
  managedEndpointNamespace: "dev_julius",
});

interface TunnelCall {
  readonly operation:
    | "get"
    | "list"
    | "create"
    | "putConfiguration"
    | "getConfiguration"
    | "getToken"
    | "delete";
  readonly input: unknown;
}

interface TunnelConfiguration {
  readonly ingress: ReadonlyArray<{
    readonly hostname?: string;
    readonly path?: string;
    readonly service: string;
  }>;
}

interface DnsCall {
  readonly operation: "listRecords" | "createRecord" | "updateRecord" | "deleteRecord";
  readonly input: unknown;
}

interface AllocationCall {
  readonly operation:
    | "get"
    | "reserve"
    | "recordTunnel"
    | "recordDns"
    | "markReady"
    | "claimRelease"
    | "claimDeprovision"
    | "removeClaimed";
  readonly input: unknown;
}

function allocationKey(input: { readonly userId: string; readonly environmentId: string }) {
  return `${input.userId}:${input.environmentId}`;
}

function makeTunnelClient(calls: TunnelCall[] = []) {
  return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
    get: (tunnelId) =>
      Effect.sync(() => {
        calls.push({ operation: "get", input: tunnelId });
        return {
          id: tunnelId,
          name: "managed-tunnel",
          status: "down",
          connsInactiveAt: "2026-06-01T00:00:00.000Z",
        };
      }),
    list: (request) =>
      Effect.sync(() => {
        calls.push({ operation: "list", input: request });
        return { result: [] };
      }),
    create: (request) =>
      Effect.sync(() => {
        calls.push({ operation: "create", input: request });
        return { id: "tunnel-id", name: request.name };
      }),
    putConfiguration: (tunnelId, tunnelConfig) =>
      Effect.sync(() => {
        calls.push({ operation: "putConfiguration", input: { tunnelId, tunnelConfig } });
      }),
    getConfiguration: (tunnelId) =>
      Effect.sync(() => {
        calls.push({ operation: "getConfiguration", input: tunnelId });
        return { ingress: [] };
      }),
    getToken: (tunnelId) =>
      Effect.sync(() => {
        calls.push({ operation: "getToken", input: tunnelId });
        return "connector-token";
      }),
    delete: (tunnelId) =>
      Effect.sync(() => {
        calls.push({ operation: "delete", input: tunnelId });
      }),
  });
}

function makePersistentTunnelClient(
  calls: TunnelCall[] = [],
  configuration: { current: TunnelConfiguration | null } = { current: null },
) {
  let tunnel: { readonly id: string; readonly name: string } | null = null;
  return ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
    get: (tunnelId) =>
      Effect.suspend(() => {
        calls.push({ operation: "get", input: tunnelId });
        return tunnel === null
          ? Effect.fail(
              new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
                operation: "get",
                tunnelId,
                cause: { _tag: "NotFound" },
              }),
            )
          : Effect.succeed({
              ...tunnel,
              status: "down",
              connsInactiveAt: "2026-06-01T00:00:00.000Z",
            });
      }),
    list: (request) =>
      Effect.sync(() => {
        calls.push({ operation: "list", input: request });
        return { result: tunnel === null ? [] : [tunnel] };
      }),
    create: (request) =>
      Effect.sync(() => {
        calls.push({ operation: "create", input: request });
        tunnel = { id: "tunnel-id", name: request.name };
        return tunnel;
      }),
    putConfiguration: (tunnelId, tunnelConfig) =>
      Effect.sync(() => {
        calls.push({ operation: "putConfiguration", input: { tunnelId, tunnelConfig } });
        configuration.current = tunnelConfig;
      }),
    getConfiguration: (tunnelId: string) =>
      Effect.sync(() => {
        calls.push({ operation: "getConfiguration", input: tunnelId });
        return configuration.current ?? { ingress: [] };
      }),
    getToken: (tunnelId) =>
      Effect.sync(() => {
        calls.push({ operation: "getToken", input: tunnelId });
        return "connector-token";
      }),
    delete: (tunnelId) =>
      Effect.sync(() => {
        calls.push({ operation: "delete", input: tunnelId });
        tunnel = null;
      }),
  });
}

function makeDnsClient(
  calls: DnsCall[] = [],
  records: ReadonlyArray<{ readonly id: string }> = [],
) {
  let currentRecords = [...records];
  return ManagedEndpointProvider.ManagedEndpointDnsClient.of({
    listRecords: (hostname) =>
      Effect.sync(() => {
        calls.push({ operation: "listRecords", input: hostname });
        return currentRecords;
      }),
    createRecord: (request) =>
      Effect.sync(() => {
        calls.push({ operation: "createRecord", input: request });
        const record = { id: "created-record-id" };
        currentRecords = [record];
        return record;
      }),
    updateRecord: (dnsRecordId, request) =>
      Effect.gen(function* () {
        calls.push({ operation: "updateRecord", input: { dnsRecordId, request } });
        if (!currentRecords.some((record) => record.id === dnsRecordId)) {
          return yield* new ManagedEndpointProvider.ManagedEndpointDnsClientError({
            operation: "update-record",
            hostname: request.name,
            dnsRecordId,
            cause: { _tag: "NotFound", dnsRecordId },
          });
        }
      }),
    deleteRecord: (dnsRecordId) =>
      Effect.sync(() => {
        calls.push({ operation: "deleteRecord", input: dnsRecordId });
        currentRecords = currentRecords.filter((record) => record.id !== dnsRecordId);
      }),
  });
}

function makeAllocations(calls: AllocationCall[] = []) {
  const allocations = new Map<string, ManagedEndpointAllocations.ManagedEndpointAllocation>();
  const recoveryEnabled = new Set<string>();
  let generation = 0;
  const mutate = (
    key: string,
    change: (
      allocation: ManagedEndpointAllocations.ManagedEndpointAllocation,
    ) => ManagedEndpointAllocations.ManagedEndpointAllocation,
  ) => {
    const allocation = allocations.get(key);
    if (allocation !== undefined) {
      allocations.set(key, {
        ...change(allocation),
        generation: allocation.generation + 1,
        updatedAt: `generation-${++generation}`,
      });
    }
  };
  return ManagedEndpointAllocations.ManagedEndpointAllocations.of({
    getByTunnelName: () => Effect.die("unused getByTunnelName"),
    get: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "get", input });
        return allocations.get(allocationKey(input)) ?? null;
      }),
    reserve: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "reserve", input });
        const allocation = allocations.get(allocationKey(input)) ?? {
          ...input,
          tunnelId: null,
          dnsRecordId: null,
          readyAt: null,
          origin: null,
          updatedAt: `generation-${++generation}`,
          generation: 0,
          tunnelReleasedAt: null,
        };
        allocations.set(allocationKey(input), allocation);
        return allocation;
      }),
    recordTunnel: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "recordTunnel", input });
        const current = allocations.get(allocationKey(input));
        if (current?.generation !== input.generation) {
          return null;
        }
        mutate(allocationKey(input), (allocation) => ({
          ...allocation,
          tunnelId: input.tunnelId,
          tunnelReleasedAt: null,
          readyAt: allocation.tunnelId === input.tunnelId ? allocation.readyAt : null,
        }));
        return allocations.get(allocationKey(input))?.generation ?? null;
      }),
    recordDns: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "recordDns", input });
        const current = allocations.get(allocationKey(input));
        if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) {
          return null;
        }
        mutate(allocationKey(input), (allocation) => ({
          ...allocation,
          dnsRecordId: input.dnsRecordId,
        }));
        return allocations.get(allocationKey(input))?.generation ?? null;
      }),
    markReady: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "markReady", input });
        const current = allocations.get(allocationKey(input));
        if (current?.generation !== input.generation || current.tunnelId !== input.tunnelId) {
          return false;
        }
        mutate(allocationKey(input), (allocation) => ({
          ...allocation,
          readyAt: "2026-06-02T00:00:00.000Z",
          origin: input.origin,
        }));
        return true;
      }),
    enableRecovery: (input) =>
      Effect.sync(() => {
        const allocation = allocations.get(allocationKey(input));
        if (allocation?.tunnelId !== input.tunnelId) {
          return false;
        }
        recoveryEnabled.add(allocationKey(input));
        return true;
      }),
    listByTunnelNames: (tunnelNames) =>
      Effect.sync(() =>
        [...allocations.values()]
          .filter((allocation) => tunnelNames.includes(allocation.tunnelName))
          .map((allocation) => ({
            ...allocation,
            recoveryEnabled: recoveryEnabled.has(allocationKey(allocation)),
          })),
      ),
    claimRelease: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "claimRelease", input });
        const allocation = allocations.get(allocationKey(input));
        if (
          allocation === undefined ||
          allocation.tunnelId !== input.tunnelId ||
          allocation.generation !== input.generation
        ) {
          return null;
        }
        mutate(allocationKey(input), (current) => ({
          ...current,
          ...(input.markReleased === true ? { tunnelReleasedAt: "2026-06-02T00:00:00.000Z" } : {}),
        }));
        return allocations.get(allocationKey(input))?.generation ?? null;
      }),
    withClaimedTunnel: (input, effect) =>
      Effect.suspend(() => {
        const current = allocations.get(allocationKey(input));
        return current?.tunnelId === input.tunnelId && current.generation === input.generation
          ? effect.pipe(
              Effect.asSome,
              Effect.onError(() =>
                Effect.sync(() => {
                  allocations.set(allocationKey(input), current);
                }),
              ),
            )
          : Effect.succeedNone;
      }),
    claimDeprovision: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "claimDeprovision", input });
        const allocation = allocations.get(allocationKey(input));
        if (allocation === undefined || allocation.generation !== input.generation) {
          return null;
        }
        mutate(allocationKey(input), (current) => current);
        return allocations.get(allocationKey(input))?.generation ?? null;
      }),
    removeClaimed: (input) =>
      Effect.sync(() => {
        calls.push({ operation: "removeClaimed", input });
        const allocation = allocations.get(allocationKey(input));
        if (allocation === undefined || allocation.generation !== input.generation) {
          return false;
        }
        allocations.delete(allocationKey(input));
        return true;
      }),
  });
}

function makeTunnelLimits(
  calls: Array<{ readonly userId: string; readonly environmentId: string }> = [],
  result: ManagedTunnelLimits.ManagedTunnelLimitExceeded | null = null,
) {
  return ManagedTunnelLimits.ManagedTunnelLimits.of({
    withCapacity: (input, reserve) =>
      Effect.gen(function* () {
        calls.push(input);
        if (result !== null) {
          return yield* result;
        }
        return yield* reserve;
      }),
    getOverride: () => Effect.die("unused"),
  });
}

function layerProvider(
  tunnelClient = makeTunnelClient(),
  dnsClient = makeDnsClient(),
  allocations = makeAllocations(),
  tunnelLimits = makeTunnelLimits(),
) {
  return ManagedEndpointProvider.layer.pipe(
    Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
    Layer.provide(RelayConfiguration.layer(config)),
    Layer.provide(ManagedEndpointProvider.layerTunnelClient(tunnelClient)),
    Layer.provide(ManagedEndpointProvider.layerDnsClient(dnsClient)),
    Layer.provide(
      Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocations),
    ),
    Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, tunnelLimits)),
  );
}

const MANAGED_ENDPOINT_HASHES: Record<string, string> = {
  "user_ABC:env_ABC": "d101ac68108a423e",
  "user_ABC:env_shared": "d7e6356aaf8863aa",
  "user_DEF:env_shared": "a2fc84ac1b8c1c35",
};

function expectedManagedEndpointHash(environmentId: string, userId: string): string {
  const hash = MANAGED_ENDPOINT_HASHES[`${userId}:${environmentId}`];
  if (hash === undefined) throw new Error(`No pinned hash for ${userId}:${environmentId}`);
  return hash;
}

function expectedManagedHostname(environmentId: string, userId = "user_ABC"): string {
  return `dev-julius-${expectedManagedEndpointHash(environmentId, userId)}.t3code.test`;
}

function expectedManagedTunnelName(environmentId: string, userId = "user_ABC"): string {
  return `t3coderelay-managedendpoint-dev-julius-${expectedManagedEndpointHash(environmentId, userId)}`;
}

describe("ManagedEndpointProvider", () => {
  it.effect("provisions a Cloudflare tunnel endpoint and connector token", () => {
    const tunnelCalls: TunnelCall[] = [];
    const dnsCalls: DnsCall[] = [];
    const allocationCalls: AllocationCall[] = [];

    return Effect.gen(function* () {
      const hostname = expectedManagedHostname("env_ABC");
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const result = yield* provider.provision({
        userId: "user_ABC",
        environmentId: "env_ABC",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      expect(result).toEqual({
        endpoint: {
          httpBaseUrl: `https://${hostname}/`,
          wsBaseUrl: `wss://${hostname}/ws`,
          providerKind: "cloudflare_tunnel",
        },
        runtime: {
          providerKind: "cloudflare_tunnel",
          connectorToken: "connector-token",
          tunnelId: "tunnel-id",
          tunnelName: expectedManagedTunnelName("env_ABC"),
        },
      });
      expect(dnsCalls).toEqual([
        { operation: "listRecords", input: hostname },
        {
          operation: "createRecord",
          input: {
            type: "CNAME",
            name: hostname,
            content: "tunnel-id.cfargotunnel.com",
            ttl: 1,
            proxied: true,
          },
        },
      ]);
      expect(tunnelCalls.map((call) => call.operation)).toEqual([
        "list",
        "create",
        "putConfiguration",
        "getToken",
      ]);
      expect(tunnelCalls[2]?.input).toMatchObject({
        tunnelConfig: {
          ingress: [
            {
              hostname,
              service: "http://127.0.0.1:3773",
            },
            { service: "http_status:404" },
          ],
        },
      });
      expect(tunnelCalls[0]?.input).toEqual({
        name: expectedManagedTunnelName("env_ABC"),
        isDeleted: false,
      });
      expect(allocationCalls.map((call) => call.operation)).toEqual([
        "reserve",
        "recordTunnel",
        "recordDns",
        "markReady",
      ]);
    }).pipe(
      Effect.provide(
        layerProvider(
          makeTunnelClient(tunnelCalls),
          makeDnsClient(dnsCalls),
          makeAllocations(allocationCalls),
        ),
      ),
    );
  });

  it.effect("refuses to provision past the managed tunnel limit without side effects", () => {
    const tunnelCalls: TunnelCall[] = [];
    const dnsCalls: DnsCall[] = [];
    const allocationCalls: AllocationCall[] = [];
    const exceeded = new ManagedTunnelLimits.ManagedTunnelLimitExceeded({
      userId: "user_ABC",
      environmentId: "env_ABC",
      maxTunnels: 10,
      activeTunnels: 10,
    });

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const error = yield* Effect.flip(
        provider.provision({
          userId: "user_ABC",
          environmentId: "env_ABC",
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        }),
      );

      expect(error).toBe(exceeded);
      expect(tunnelCalls).toEqual([]);
      expect(dnsCalls).toEqual([]);
      expect(allocationCalls).toEqual([]);
    }).pipe(
      Effect.provide(
        layerProvider(
          makeTunnelClient(tunnelCalls),
          makeDnsClient(dnsCalls),
          makeAllocations(allocationCalls),
          makeTunnelLimits([], exceeded),
        ),
      ),
    );
  });

  it.effect("uses stage-scoped stable names without leaking unusual environment ids", () => {
    const tunnelCalls: TunnelCall[] = [];

    return Effect.gen(function* () {
      const environmentId = "ENV With Spaces/../Symbols!" + "x".repeat(80);
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      yield* provider.provision({
        userId: "user_ABC",
        environmentId,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      const requestedName = (
        tunnelCalls.find((call) => call.operation === "list")?.input as
          | { readonly name?: string }
          | undefined
      )?.name;
      expect(requestedName).toMatch(/^t3coderelay-managedendpoint-dev-julius-[a-f0-9]{16}$/);
      const configBody = (
        tunnelCalls.find((call) => call.operation === "putConfiguration")?.input as
          | { readonly tunnelConfig?: unknown }
          | undefined
      )?.tunnelConfig;
      expect(configBody).toMatchObject({
        ingress: [
          {
            hostname: expect.stringMatching(/^dev-julius-[a-f0-9]{16}\.t3code\.test$/),
          },
          { service: "http_status:404" },
        ],
      });
      const hostname = (
        configBody as
          | {
              readonly ingress?: readonly [{ readonly hostname?: unknown }, unknown];
            }
          | undefined
      )?.ingress?.[0]?.hostname;
      expect(typeof hostname === "string" ? hostname.split(".")[0]?.length : 0).toBeLessThanOrEqual(
        63,
      );
      expect(tunnelCalls.find((call) => call.operation === "create")?.input).toMatchObject({
        name: requestedName,
        configSrc: "cloudflare",
      });
    }).pipe(Effect.provide(layerProvider(makeTunnelClient(tunnelCalls))));
  });

  it.effect("formats IPv6 loopback origins as valid Cloudflare ingress service URLs", () => {
    const tunnelCalls: TunnelCall[] = [];

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      yield* provider.provision({
        userId: "user_ABC",
        environmentId: "env-ipv6",
        origin: { localHttpHost: "::1", localHttpPort: 3773 },
      });

      expect(
        tunnelCalls.find((call) => call.operation === "putConfiguration")?.input,
      ).toMatchObject({
        tunnelConfig: {
          ingress: [
            {
              service: "http://[::1]:3773",
            },
            { service: "http_status:404" },
          ],
        },
      });
    }).pipe(Effect.provide(layerProvider(makeTunnelClient(tunnelCalls))));
  });

  it.effect("rejects non-loopback managed endpoint origins before calling Cloudflare", () => {
    const dnsCalls: DnsCall[] = [];

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const result = yield* Effect.result(
        provider.provision({
          userId: "user_ABC",
          environmentId: "env_ABC",
          origin: { localHttpHost: "192.168.1.10", localHttpPort: 3773 },
        }),
      );

      expect(dnsCalls).toHaveLength(0);
      expect(result._tag).toBe("Failure");
      if (result._tag === "Failure") {
        expect(result.failure).toMatchObject({
          _tag: "ManagedEndpointOriginNotAllowed",
          userId: "user_ABC",
          environmentId: "env_ABC",
          host: "192.168.1.10",
          port: 3773,
        });
      }
    }).pipe(Effect.provide(layerProvider(makeTunnelClient(), makeDnsClient(dnsCalls))));
  });

  it.effect("reuses checkpointed resources when provisioning is retried", () => {
    const tunnelCalls: TunnelCall[] = [];
    const dnsCalls: DnsCall[] = [];
    const allocationCalls: AllocationCall[] = [];
    const layer = layerProvider(
      makePersistentTunnelClient(tunnelCalls),
      makeDnsClient(dnsCalls),
      makeAllocations(allocationCalls),
    );

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const request = {
        userId: "user_ABC",
        environmentId: "env_ABC",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      } as const;
      yield* provider.provision(request);
      yield* provider.provision(request);

      expect(tunnelCalls.map((call) => call.operation)).toEqual([
        "list",
        "create",
        "putConfiguration",
        "getToken",
        "list",
        "putConfiguration",
        "getToken",
      ]);
      expect(dnsCalls.map((call) => call.operation)).toEqual([
        "listRecords",
        "createRecord",
        "updateRecord",
      ]);
      expect(allocationCalls.map((call) => call.operation)).toEqual([
        "reserve",
        "recordTunnel",
        "recordDns",
        "markReady",
        "reserve",
        "recordTunnel",
        "recordDns",
        "markReady",
      ]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("recovers a ready tunnel for the same origin with only reads", () => {
    const tunnelCalls: TunnelCall[] = [];
    const dnsCalls: DnsCall[] = [];
    const allocationCalls: AllocationCall[] = [];
    const layer = layerProvider(
      makePersistentTunnelClient(tunnelCalls),
      makeDnsClient(dnsCalls),
      makeAllocations(allocationCalls),
    );

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const request = {
        userId: "user_ABC",
        environmentId: "env_ABC",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      } as const;
      const provisioned = yield* provider.provision(request);
      tunnelCalls.length = 0;
      dnsCalls.length = 0;
      allocationCalls.length = 0;

      const recovered = yield* provider.provision({ ...request, reuseReadyTunnel: true });

      expect(recovered).toEqual(provisioned);
      expect(tunnelCalls.map((call) => call.operation)).toEqual(["get", "getToken"]);
      expect(dnsCalls).toEqual([]);
      expect(allocationCalls.map((call) => call.operation)).toEqual(["reserve"]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("fully provisions a recovery when the ready tunnel cannot be reused", () => {
    const tunnelCalls: TunnelCall[] = [];
    const tunnelClient = makePersistentTunnelClient(tunnelCalls);
    const layer = layerProvider(tunnelClient);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const request = {
        userId: "user_ABC",
        environmentId: "env_ABC",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        reuseReadyTunnel: true,
      } as const;
      yield* provider.provision(request);

      tunnelCalls.length = 0;
      yield* provider.provision({
        ...request,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3774 },
      });
      expect(tunnelCalls.map((call) => call.operation)).toEqual([
        "list",
        "putConfiguration",
        "getToken",
      ]);

      yield* tunnelClient.delete("tunnel-id");
      tunnelCalls.length = 0;
      yield* provider.provision({
        ...request,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3774 },
      });
      expect(tunnelCalls.map((call) => call.operation)).toEqual([
        "get",
        "list",
        "create",
        "putConfiguration",
        "getToken",
      ]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("keeps a pause recorded while a shared pause read was in flight", () => {
    let listCalls = 0;
    const tunnelClient = {
      list: () =>
        Effect.suspend(() => {
          listCalls += 1;
          return Effect.fail({ _tag: "TooManyRequests", message: "Rate limited" });
        }),
    } as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient;

    return Effect.gen(function* () {
      const readStarted = yield* Deferred.make<void>();
      const releaseRead = yield* Deferred.make<void>();
      const layer = ManagedEndpointProvider.layerCloudflareBindings(
        tunnelClient,
        {} as Cloudflare.DNS.ReadWriteDnsClient,
        {} as Alchemy.BaseRuntimeContext,
      ).pipe(
        Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
        Layer.provide(RelayConfiguration.layer(config)),
        Layer.provide(
          Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()),
        ),
        Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())),
        Layer.provide(
          Layer.succeed(
            CloudflareApiPause.CloudflareApiPause,
            CloudflareApiPause.CloudflareApiPause.of({
              pausedUntilMillis: Deferred.succeed(readStarted, undefined).pipe(
                Effect.andThen(Deferred.await(releaseRead)),
                Effect.as(0),
              ),
              pauseUntil: () => Effect.void,
            }),
          ),
        ),
      );

      yield* Effect.gen(function* () {
        const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
        const list = tunnels.list({ isDeleted: false }).pipe(Effect.flip);
        // The first call starts the shared read; the second hits a 429 meanwhile.
        const reading = yield* Effect.forkChild(list);
        yield* Deferred.await(readStarted);
        yield* list;
        yield* Deferred.succeed(releaseRead, undefined);
        yield* Fiber.join(reading);
        yield* list;
        expect(listCalls).toBe(1);
      }).pipe(Effect.provide(layer));
    });
  });

  it.effect("reads a tunnel without remote configuration as having no ingress", () => {
    const tunnelClient = {
      getConfiguration: () =>
        Effect.fail(
          new ZeroTrust.TunnelConfigurationNotFound({
            code: 1055,
            message: "Tunnel configuration not found",
          }),
        ),
    } as unknown as Cloudflare.Tunnel.ReadWriteTunnelClient;
    const layer = ManagedEndpointProvider.layerCloudflareBindings(
      tunnelClient,
      {} as Cloudflare.DNS.ReadWriteDnsClient,
      {} as Alchemy.BaseRuntimeContext,
    ).pipe(
      Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
      Layer.provide(RelayConfiguration.layer(config)),
      Layer.provide(
        Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, makeAllocations()),
      ),
      Layer.provide(Layer.succeed(ManagedTunnelLimits.ManagedTunnelLimits, makeTunnelLimits())),
      Layer.provide(
        Layer.succeed(
          CloudflareApiPause.CloudflareApiPause,
          CloudflareApiPause.CloudflareApiPause.of({
            pausedUntilMillis: Effect.succeed(0),
            pauseUntil: () => Effect.void,
          }),
        ),
      ),
    );

    return Effect.gen(function* () {
      const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
      expect(yield* tunnels.getConfiguration("tunnel-id")).toEqual({ ingress: [] });
    }).pipe(Effect.provide(layer));
  });

  it.effect("does not hide non-not-found checkpoint update failures", () => {
    const dnsCalls: DnsCall[] = [];
    const failure = new ManagedEndpointProvider.ManagedEndpointDnsClientError({
      operation: "update-record",
      dnsRecordId: "created-record-id",
      cause: new Error("Cloudflare DNS unavailable"),
    });
    let records: ReadonlyArray<{ readonly id: string }> = [];
    const dnsClient = ManagedEndpointProvider.ManagedEndpointDnsClient.of({
      listRecords: (hostname) =>
        Effect.sync(() => {
          dnsCalls.push({ operation: "listRecords", input: hostname });
          return records;
        }),
      createRecord: (request) =>
        Effect.sync(() => {
          dnsCalls.push({ operation: "createRecord", input: request });
          const record = { id: "created-record-id" };
          records = [record];
          return record;
        }),
      updateRecord: (dnsRecordId, request) =>
        Effect.sync(() => {
          dnsCalls.push({ operation: "updateRecord", input: { dnsRecordId, request } });
        }).pipe(Effect.andThen(Effect.fail(failure))),
      deleteRecord: () => Effect.void,
    });
    const allocations = makeAllocations();
    const layer = layerProvider(makePersistentTunnelClient(), dnsClient, allocations);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const request = {
        userId: "user_ABC",
        environmentId: "env_ABC",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      } as const;
      yield* provider.provision(request);
      const error = yield* Effect.flip(provider.provision(request));

      expect(error).toMatchObject({
        _tag: "ManagedEndpointProvisioningFailed",
        stage: "ensure-dns-record",
        userId: "user_ABC",
        environmentId: "env_ABC",
      });
      expect(dnsCalls.map((call) => call.operation)).toEqual([
        "listRecords",
        "createRecord",
        "updateRecord",
      ]);
      expect(yield* allocations.get(request)).toMatchObject({
        tunnelId: "tunnel-id",
        readyAt: "2026-06-02T00:00:00.000Z",
      });
    }).pipe(Effect.provide(layer));
  });

  it.effect(
    "deprovisions checkpointed DNS and tunnel resources before removing the allocation",
    () => {
      const tunnelCalls: TunnelCall[] = [];
      const dnsCalls: DnsCall[] = [];
      const allocationCalls: AllocationCall[] = [];
      const layer = layerProvider(
        makePersistentTunnelClient(tunnelCalls),
        makeDnsClient(dnsCalls),
        makeAllocations(allocationCalls),
      );

      return Effect.gen(function* () {
        const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
        const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
        yield* provider.provision({
          ...key,
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        });
        yield* provider.deprovision(key);

        expect(dnsCalls.map((call) => call.operation)).toEqual([
          "listRecords",
          "createRecord",
          "deleteRecord",
        ]);
        expect(tunnelCalls.map((call) => call.operation)).toEqual([
          "list",
          "create",
          "putConfiguration",
          "getToken",
          "delete",
        ]);
        expect(allocationCalls.map((call) => call.operation)).toEqual([
          "reserve",
          "recordTunnel",
          "recordDns",
          "markReady",
          "get",
          "claimDeprovision",
          "removeClaimed",
        ]);
      }).pipe(Effect.provide(layer));
    },
  );

  it.effect("does not deprovision an allocation superseded by a concurrent relink", () => {
    const tunnelCalls: TunnelCall[] = [];
    const dnsCalls: DnsCall[] = [];
    const allocationCalls: AllocationCall[] = [];
    const layer = layerProvider(
      makePersistentTunnelClient(tunnelCalls),
      makeDnsClient(dnsCalls),
      makeAllocations(allocationCalls),
    );

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      const request = {
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      } as const;
      yield* provider.provision(request);
      const unlinkTarget = yield* provider.prepareDeprovision(key);
      expect(unlinkTarget).not.toBeNull();
      if (unlinkTarget === null) {
        return;
      }

      yield* provider.provision(request);
      const tunnelCallCount = tunnelCalls.length;
      const dnsCallCount = dnsCalls.length;
      const allocationCallCount = allocationCalls.length;

      yield* provider.deprovision({ ...key, target: unlinkTarget });

      expect(tunnelCalls).toHaveLength(tunnelCallCount);
      expect(dnsCalls).toHaveLength(dnsCallCount);
      expect(allocationCalls.slice(allocationCallCount).map((call) => call.operation)).toEqual([
        "claimDeprovision",
      ]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("releases the tunnel while keeping the allocation, DNS record, and hostname", () => {
    const tunnelCalls: TunnelCall[] = [];
    const dnsCalls: DnsCall[] = [];
    const allocationCalls: AllocationCall[] = [];
    const layer = layerProvider(
      makePersistentTunnelClient(tunnelCalls),
      makeDnsClient(dnsCalls),
      makeAllocations(allocationCalls),
    );

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
      const first = yield* provider.provision({ ...key, origin });
      const released = yield* provider.release(key);
      const second = yield* provider.provision({ ...key, origin });

      expect(released).toBe(true);
      expect(second.endpoint).toEqual(first.endpoint);
      expect(tunnelCalls.map((call) => call.operation)).toEqual([
        "list",
        "create",
        "putConfiguration",
        "getToken",
        "delete",
        "list",
        "create",
        "putConfiguration",
        "getToken",
      ]);
      expect(dnsCalls.map((call) => call.operation)).toEqual([
        "listRecords",
        "createRecord",
        "updateRecord",
      ]);
      const claims = allocationCalls.filter((call) => call.operation === "claimRelease");
      expect(claims.map((call) => (call.input as { markReleased?: boolean }).markReleased)).toEqual(
        [undefined, undefined],
      );
    }).pipe(Effect.provide(layer));
  });

  it.effect("marks the release on the claim that deletes the tunnel when asked", () => {
    const allocations = makeAllocations();
    const layer = layerProvider(makePersistentTunnelClient(), makeDnsClient(), allocations);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      expect(yield* provider.release({ ...key, markReleased: true })).toBe(true);
      expect((yield* allocations.get(key))?.tunnelReleasedAt).toBe("2026-06-02T00:00:00.000Z");
    }).pipe(Effect.provide(layer));
  });

  it.effect("keeps the tunnel alive when a concurrent provision outdates the release claim", () => {
    const tunnelCalls: TunnelCall[] = [];
    const allocations = makeAllocations();
    const outdated = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
      ...allocations,
      claimRelease: () => Effect.succeed(null),
    });
    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), outdated);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      const released = yield* provider.release(key);

      expect(released).toBe(false);
      expect(tunnelCalls.map((call) => call.operation)).toEqual([
        "list",
        "create",
        "putConfiguration",
        "getToken",
      ]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("does not release a tunnel when the requested tunnel id is outdated", () => {
    const tunnelCalls: TunnelCall[] = [];
    const layer = layerProvider(
      makePersistentTunnelClient(tunnelCalls),
      makeDnsClient(),
      makeAllocations(),
    );

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      expect(yield* provider.release({ ...key, expectedTunnelId: "old-tunnel-id" })).toBe(false);
      expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
    }).pipe(Effect.provide(layer));
  });

  it.effect("keeps a tunnel when a provision replaces an ordinary release generation", () => {
    const tunnelCalls: TunnelCall[] = [];
    const allocations = makeAllocations();
    let replaceAfterClaim = false;
    const replaced = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
      ...allocations,
      claimRelease: (input) =>
        allocations.claimRelease(input).pipe(
          Effect.tap((claimedGeneration) => {
            if (claimedGeneration === null || replaceAfterClaim) return Effect.void;
            replaceAfterClaim = true;
            return allocations
              .recordTunnel({
                userId: input.userId,
                environmentId: input.environmentId,
                tunnelId: "replacement-tunnel",
                generation: claimedGeneration,
              })
              .pipe(Effect.asVoid);
          }),
        ),
    });
    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), replaced);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      expect(yield* provider.release(key)).toBe(false);
      expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
    }).pipe(Effect.provide(layer));
  });

  it.effect("asks for recovery when the recorded tunnel was deleted", () => {
    const tunnelCalls: TunnelCall[] = [];
    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls));

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
      const provisioned = yield* provider.provision({ ...key, origin });
      expect(yield* provider.release(key)).toBe(true);

      expect(
        yield* provider.reconcileOrigin({
          ...key,
          tunnelId: provisioned.runtime.tunnelId!,
          origin,
          endpoint: provisioned.endpoint,
        }),
      ).toBe("recovery_required");
    }).pipe(Effect.provide(layer));
  });

  it.effect("updates Cloudflare ingress once when the registered port changes", () => {
    const tunnelCalls: TunnelCall[] = [];
    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls));

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      const provisioned = yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      tunnelCalls.length = 0;

      expect(
        yield* provider.reconcileOrigin({
          ...key,
          tunnelId: provisioned.runtime.tunnelId!,
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 4884 },
          endpoint: provisioned.endpoint,
        }),
      ).toBe("ready");
      expect(tunnelCalls).toEqual([
        { operation: "get", input: "tunnel-id" },
        {
          operation: "putConfiguration",
          input: {
            tunnelId: "tunnel-id",
            tunnelConfig: {
              ingress: [
                {
                  hostname: expectedManagedHostname("env_ABC"),
                  service: "http://127.0.0.1:4884",
                },
                { service: "http_status:404" },
              ],
            },
          },
        },
      ]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("restores Cloudflare ingress that drifted from the recorded origin", () => {
    const tunnelCalls: TunnelCall[] = [];
    const configuration: { current: TunnelConfiguration | null } = { current: null };
    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls, configuration));

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      const origin = { localHttpHost: "127.0.0.1", localHttpPort: 3773 } as const;
      const provisioned = yield* provider.provision({ ...key, origin });
      const reconcile = provider.reconcileOrigin({
        ...key,
        tunnelId: provisioned.runtime.tunnelId!,
        origin,
        endpoint: provisioned.endpoint,
      });
      tunnelCalls.length = 0;

      expect(yield* reconcile).toBe("ready");
      expect(tunnelCalls.map((call) => call.operation)).toEqual(["get", "getConfiguration"]);

      configuration.current = {
        ingress: [
          { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:4711" },
          { service: "http_status:404" },
        ],
      };
      expect(yield* reconcile).toBe("ready");
      expect(configuration.current).toEqual({
        ingress: [
          { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:3773" },
          { service: "http_status:404" },
        ],
      });

      configuration.current = {
        ingress: [
          { hostname: "*.t3code.test", service: "http://127.0.0.1:4711" },
          { hostname: expectedManagedHostname("env_ABC"), service: "http://127.0.0.1:3773" },
          { service: "http_status:404" },
        ],
      };
      expect(yield* reconcile).toBe("ready");
      expect(configuration.current.ingress[0]).toEqual({
        hostname: expectedManagedHostname("env_ABC"),
        service: "http://127.0.0.1:3773",
      });

      configuration.current = {
        ingress: [
          {
            hostname: expectedManagedHostname("env_ABC"),
            path: "/api",
            service: "http://127.0.0.1:3773",
          },
          { service: "http_status:404" },
        ],
      };
      expect(yield* reconcile).toBe("ready");
      expect(configuration.current.ingress[0]).toEqual({
        hostname: expectedManagedHostname("env_ABC"),
        service: "http://127.0.0.1:3773",
      });
    }).pipe(Effect.provide(layer));
  });

  it.effect("rejects an active endpoint that does not match the allocation hostname", () => {
    const layer = layerProvider(makePersistentTunnelClient());

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      const provisioned = yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      const error = yield* Effect.flip(
        provider.reconcileOrigin({
          ...key,
          tunnelId: provisioned.runtime.tunnelId!,
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
          endpoint: {
            ...provisioned.endpoint,
            httpBaseUrl: "https://different-host.t3code.test/",
          },
        }),
      );
      expect(error).toMatchObject({
        _tag: "ManagedEndpointProvisioningFailed",
        stage: "verify-endpoint",
      });
    }).pipe(Effect.provide(layer));
  });

  it.effect(
    "keeps a newly created tunnel available for retry after losing its allocation claim",
    () => {
      const tunnelCalls: TunnelCall[] = [];
      const allocations = makeAllocations();
      let changeGeneration = true;
      const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
        ...allocations,
        recordTunnel: (input) =>
          Effect.gen(function* () {
            if (changeGeneration) {
              changeGeneration = false;
              yield* allocations.claimDeprovision(input);
            }
            return yield* allocations.recordTunnel(input);
          }),
      });
      const layer = layerProvider(
        makePersistentTunnelClient(tunnelCalls),
        makeDnsClient(),
        changed,
      );

      return Effect.gen(function* () {
        const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
        const input = {
          userId: "user_ABC",
          environmentId: "env_ABC",
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        };
        const error = yield* Effect.flip(provider.provision(input));

        expect(error).toMatchObject({
          _tag: "ManagedEndpointProvisioningFailed",
          stage: "record-tunnel",
          reason: "claim-lost",
        });
        expect(tunnelCalls.map((call) => call.operation)).toEqual(["list", "create"]);
        expect((yield* provider.provision(input)).runtime.tunnelId).toBe("tunnel-id");
        expect(tunnelCalls.filter((call) => call.operation === "create")).toHaveLength(1);
        expect(tunnelCalls.filter((call) => call.operation === "delete")).toEqual([]);
      }).pipe(Effect.provide(layer));
    },
  );

  it.effect("does not change tunnel ingress after another provision takes ownership", () => {
    const tunnelCalls: TunnelCall[] = [];
    const allocations = makeAllocations();
    const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
      ...allocations,
      withClaimedTunnel: () => Effect.succeedNone,
    });
    const layer = layerProvider(makePersistentTunnelClient(tunnelCalls), makeDnsClient(), changed);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const error = yield* Effect.flip(
        provider.provision({
          userId: "user_ABC",
          environmentId: "env_ABC",
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        }),
      );

      expect(error).toMatchObject({
        _tag: "ManagedEndpointProvisioningFailed",
        stage: "configure-tunnel",
        reason: "claim-lost",
      });
      expect(tunnelCalls.map((call) => call.operation)).not.toContain("putConfiguration");
    }).pipe(Effect.provide(layer));
  });

  it.effect("does not change DNS after another provision takes ownership", () => {
    const dnsCalls: DnsCall[] = [];
    const allocations = makeAllocations();
    let lockCount = 0;
    const changed = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
      ...allocations,
      withClaimedTunnel: (input, effect) =>
        ++lockCount === 1 ? allocations.withClaimedTunnel(input, effect) : Effect.succeedNone,
    });
    const layer = layerProvider(makePersistentTunnelClient(), makeDnsClient(dnsCalls), changed);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const error = yield* Effect.flip(
        provider.provision({
          userId: "user_ABC",
          environmentId: "env_ABC",
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        }),
      );

      expect(error).toMatchObject({
        _tag: "ManagedEndpointProvisioningFailed",
        stage: "record-dns",
      });
      expect(dnsCalls).toEqual([]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("keeps a tunnel that reconnects before scheduled deletion", () => {
    const tunnelCalls: TunnelCall[] = [];
    const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
      ...makePersistentTunnelClient(tunnelCalls),
      get: (tunnelId) =>
        Effect.succeed({
          id: tunnelId,
          name: expectedManagedTunnelName("env_ABC"),
          status: "healthy",
          connsInactiveAt: null,
        }),
    });
    const layer = layerProvider(tunnelClient, makeDnsClient(), makeAllocations());

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      expect(
        yield* provider.release({
          ...key,
          expectedTunnelId: "tunnel-id",
          expectedStatus: "down",
          expectedInactiveBefore: "2026-06-01T00:05:00.000Z",
        }),
      ).toBe(false);
      expect(tunnelCalls.map((call) => call.operation)).not.toContain("delete");
    }).pipe(Effect.provide(layer));
  });

  it.effect.each([
    { name: "the tunnel is gone", tunnelRemoved: true, released: true },
    { name: "the tunnel still exists", tunnelRemoved: false, released: false },
  ] as const)(
    "resolves a delete whose response was lost by checking whether $name",
    ({ tunnelRemoved, released }) => {
      const persistent = makePersistentTunnelClient();
      const lostResponse = new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
        operation: "delete",
        tunnelId: "tunnel-id",
        cause: { _tag: "TimeoutError" },
      });
      const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
        ...persistent,
        delete: (tunnelId) =>
          (tunnelRemoved ? persistent.delete(tunnelId) : Effect.void).pipe(
            Effect.andThen(Effect.fail(lostResponse)),
          ),
      });
      const allocationCalls: AllocationCall[] = [];
      const allocations = makeAllocations(allocationCalls);
      const layer = layerProvider(tunnelClient, makeDnsClient(), allocations);

      return Effect.gen(function* () {
        const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
        const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
        yield* provider.provision({
          ...key,
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        });
        const result = yield* Effect.result(provider.release({ ...key, markReleased: true }));
        expect((yield* allocations.get(key))?.tunnelReleasedAt).toBe(
          released ? "2026-06-02T00:00:00.000Z" : null,
        );

        if (released) {
          expect(result).toMatchObject({ _tag: "Success", success: true });
        } else {
          expect(result).toMatchObject({
            _tag: "Failure",
            failure: { stage: "delete-tunnel", cause: lostResponse },
          });
        }
      }).pipe(Effect.provide(layer));
    },
  );

  it.effect("reports a tunnel with an attached connector as not released", () => {
    const persistent = makePersistentTunnelClient();
    const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
      ...persistent,
      delete: (tunnelId) =>
        Effect.fail(
          new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
            operation: "delete",
            tunnelId,
            cause: {
              _tag: "BadRequest",
              code: 1022,
              message:
                "This tunnel has active connections. Please stop all cloudflared replicas, or wait a few minutes for connections to close, then try again.",
            },
          }),
        ),
    });
    const allocations = makeAllocations();
    const layer = layerProvider(tunnelClient, makeDnsClient(), allocations);

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      expect(yield* provider.release({ ...key, markReleased: true })).toBe(false);
      expect((yield* allocations.get(key))?.tunnelReleasedAt).toBeNull();
      expect((yield* persistent.get("tunnel-id")).id).toBe("tunnel-id");
    }).pipe(Effect.provide(layer));
  });

  it.effect("keeps the allocation when tunnel cleanup fails so unlink can retry", () => {
    const allocationCalls: AllocationCall[] = [];
    const tunnelCalls: TunnelCall[] = [];
    let deleteAttempts = 0;
    const failure = new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
      operation: "delete",
      tunnelId: "tunnel-id",
      cause: "Cloudflare tunnel deletion failed",
    });
    const tunnels = makePersistentTunnelClient(tunnelCalls);
    const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
      ...tunnels,
      delete: (tunnelId) =>
        Effect.gen(function* () {
          tunnelCalls.push({ operation: "delete", input: tunnelId });
          deleteAttempts++;
          if (deleteAttempts === 1) {
            return yield* failure;
          }
        }),
    });
    const layer = layerProvider(tunnelClient, makeDnsClient(), makeAllocations(allocationCalls));

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      const first = yield* Effect.result(provider.deprovision(key));
      expect(first._tag).toBe("Failure");
      if (first._tag === "Failure") {
        expect(first.failure).toMatchObject({
          _tag: "ManagedEndpointDeprovisioningFailed",
          stage: "delete-tunnel",
          userId: key.userId,
          environmentId: key.environmentId,
          tunnelId: "tunnel-id",
        });
        expect(first.failure.cause).toBe(failure);
      }
      yield* provider.deprovision(key);

      expect(allocationCalls.map((call) => call.operation)).toEqual([
        "reserve",
        "recordTunnel",
        "recordDns",
        "markReady",
        "get",
        "claimDeprovision",
        "get",
        "claimDeprovision",
        "removeClaimed",
      ]);
    }).pipe(Effect.provide(layer));
  });

  it.effect("marks the tunnel released when removing the allocation fails after deletion", () => {
    const allocationCalls: AllocationCall[] = [];
    const allocations = makeAllocations(allocationCalls);
    const layer = layerProvider(makePersistentTunnelClient(), makeDnsClient(), {
      ...allocations,
      removeClaimed: (input) =>
        Effect.suspend(() => {
          allocationCalls.push({ operation: "removeClaimed", input });
          return Effect.fail(
            new ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError({
              operation: "remove-claimed",
              stage: "database-request",
              userId: input.userId,
              environmentId: input.environmentId,
            }),
          );
        }),
    });

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      const result = yield* Effect.result(provider.deprovision(key));

      expect(result._tag).toBe("Failure");
      if (result._tag === "Failure") {
        expect(result.failure).toMatchObject({ stage: "remove-allocation" });
      }
      // The tunnel is gone; without the mark the tunnel limit would keep counting it.
      const claim = allocationCalls.find((call) => call.operation === "claimDeprovision");
      if (claim === undefined) return expect.unreachable("deprovision never claimed the row");
      expect(allocationCalls.at(-1)).toMatchObject({
        operation: "claimRelease",
        input: {
          tunnelId: "tunnel-id",
          generation: (claim.input as { generation: number }).generation + 1,
          markReleased: true,
        },
      });
    }).pipe(Effect.provide(layer));
  });

  it.effect("treats already deleted remote resources as successfully deprovisioned", () => {
    const allocationCalls: AllocationCall[] = [];
    const notFound = { _tag: "NotFound" } as const;
    const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
      ...makeTunnelClient(),
      delete: () =>
        Effect.fail(
          new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
            operation: "delete",
            tunnelId: "tunnel-id",
            cause: notFound,
          }),
        ),
    });
    const dnsClient = ManagedEndpointProvider.ManagedEndpointDnsClient.of({
      ...makeDnsClient(),
      deleteRecord: () =>
        Effect.fail(
          new ManagedEndpointProvider.ManagedEndpointDnsClientError({
            operation: "delete-record",
            dnsRecordId: "created-record-id",
            cause: notFound,
          }),
        ),
    });
    const layer = layerProvider(tunnelClient, dnsClient, makeAllocations(allocationCalls));

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const key = { userId: "user_ABC", environmentId: "env_ABC" } as const;
      yield* provider.provision({
        ...key,
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      yield* provider.deprovision(key);

      expect(allocationCalls.map((call) => call.operation)).toContain("removeClaimed");
    }).pipe(Effect.provide(layer));
  });

  it.effect("scopes managed endpoint resources by user", () => {
    const tunnelCalls: TunnelCall[] = [];

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      yield* provider.provision({
        userId: "user_ABC",
        environmentId: "env_shared",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });
      yield* provider.provision({
        userId: "user_DEF",
        environmentId: "env_shared",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      expect(
        tunnelCalls.filter((call) => call.operation === "list").map((call) => call.input),
      ).toEqual([
        { name: expectedManagedTunnelName("env_shared", "user_ABC"), isDeleted: false },
        { name: expectedManagedTunnelName("env_shared", "user_DEF"), isDeleted: false },
      ]);
    }).pipe(Effect.provide(layerProvider(makeTunnelClient(tunnelCalls))));
  });

  it.effect("recovers when DNS creation reports failure after the record became visible", () => {
    const dnsCalls: DnsCall[] = [];
    const failure = new ManagedEndpointProvider.ManagedEndpointDnsClientError({
      operation: "create-record",
      hostname: expectedManagedHostname("env_ABC"),
      cause: "ambiguous Cloudflare DNS response",
    });
    let records: ReadonlyArray<{ readonly id: string }> = [];
    const dnsClient = ManagedEndpointProvider.ManagedEndpointDnsClient.of({
      listRecords: (hostname) =>
        Effect.sync(() => {
          dnsCalls.push({ operation: "listRecords", input: hostname });
          return records;
        }),
      createRecord: (request) =>
        Effect.gen(function* () {
          dnsCalls.push({ operation: "createRecord", input: request });
          records = [{ id: "created-record-id" }];
          return yield* failure;
        }),
      updateRecord: (dnsRecordId, request) =>
        Effect.sync(() => {
          dnsCalls.push({ operation: "updateRecord", input: { dnsRecordId, request } });
        }),
      deleteRecord: (dnsRecordId) =>
        Effect.sync(() => {
          dnsCalls.push({ operation: "deleteRecord", input: dnsRecordId });
        }),
    });

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      yield* provider.provision({
        userId: "user_ABC",
        environmentId: "env_ABC",
        origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
      });

      expect(dnsCalls.map((call) => call.operation)).toEqual([
        "listRecords",
        "createRecord",
        "listRecords",
        "updateRecord",
      ]);
    }).pipe(Effect.provide(layerProvider(makeTunnelClient(), dnsClient)));
  });

  it.effect("reports mismatched tunnel responses without manufacturing a cause", () => {
    const dnsCalls: DnsCall[] = [];
    const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
      ...makeTunnelClient(),
      create: () => Effect.succeed({ id: "returned-tunnel-id", name: "unexpected-tunnel" }),
    });

    return Effect.gen(function* () {
      const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
      const error = yield* Effect.flip(
        provider.provision({
          userId: "user_ABC",
          environmentId: "env_ABC",
          origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
        }),
      );

      expect(error).toMatchObject({
        _tag: "ManagedEndpointProvisioningFailed",
        stage: "validate-tunnel-response",
        userId: "user_ABC",
        environmentId: "env_ABC",
        hostname: expectedManagedHostname("env_ABC"),
        tunnelName: expectedManagedTunnelName("env_ABC"),
        returnedTunnelId: "returned-tunnel-id",
        returnedTunnelName: "unexpected-tunnel",
      });
      if (error._tag === "ManagedEndpointProvisioningFailed") {
        expect(error.cause).toBeUndefined();
      }
      expect(dnsCalls).toHaveLength(0);
    }).pipe(Effect.provide(layerProvider(tunnelClient, makeDnsClient(dnsCalls))));
  });
});