sailfork / t3code-github / apps/ server/ src/ cloud/ ManagedEndpointRuntime.ts

import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
import { RotatingFileSink, RotatingFileSinkError } from "@t3tools/shared/logging";
import * as HostProcess from "@t3tools/shared/HostProcess";
import * as RelayClient from "@t3tools/shared/relayClient";
import * as Clock from "effect/Clock";
import * as Config from "effect/Config";
import * as Context from "effect/Context";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Layer from "effect/Layer";
import * as Logger from "effect/Logger";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Queue from "effect/Queue";
import * as Ref from "effect/Ref";
import * as Result from "effect/Result";
import * as Schema from "effect/Schema";
import * as Semaphore from "effect/Semaphore";
import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
import { HttpClient, HttpClientResponse } from "effect/http";
import * as ChildProcess from "effect/process/ChildProcess";
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";

import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
import * as ServerConfig from "../config.ts";
import * as ManagedLimits from "../policy/managedLimits.ts";
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";

export type CloudManagedEndpointRuntimeStatus =
  | {
      readonly status: "disabled";
    }
  | {
      readonly status: "failed";
      readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];
      readonly failure: "unsupported-platform" | "not-installed" | "spawn-failed";
      readonly reason: string;
      readonly tunnelId?: string;
      readonly tunnelName?: string;
    }
  | {
      readonly status: "running";
      readonly providerKind: "cloudflare_tunnel";
      readonly pid: number;
      readonly tunnelId?: string;
      readonly tunnelName?: string;
    }
  | {
      readonly status: "unsupported";
      readonly providerKind: RelayManagedEndpointRuntimeConfig["providerKind"];
    };

/** What the running cloudflared reports about itself. `null` fields are not known yet. */
export interface ConnectorFacts {
  readonly pid: number;
  readonly readyConnections: number | null;
  /** The public tunnel hostname from the remote config cloudflared receives once it connects. */
  readonly hostname: string | null;
  readonly protocol: string | null;
}

export class CloudManagedEndpointRuntime extends Context.Service<
  CloudManagedEndpointRuntime,
  {
    readonly applyConfig: (
      config: RelayManagedEndpointRuntimeConfig | null,
    ) => Effect.Effect<CloudManagedEndpointRuntimeStatus>;
    /** `null` when no connector is running. */
    readonly inspectConnector: Effect.Effect<ConnectorFacts | null>;
    /** Emits when a connector registers a connection or its readiness changes. */
    readonly readinessChanged: Stream.Stream<void>;
    readonly recoveryRequests: Stream.Stream<RelayManagedEndpointRuntimeConfig>;
    readonly requestRecovery: (config: RelayManagedEndpointRuntimeConfig) => Effect.Effect<void>;
    /**
     * Restarts the connector if it still runs as `pid` and `stillDead` holds once it has
     * the lock. Returns whether it sent the restart.
     */
    readonly restartConnector: (
      pid: number,
      stillDead: Effect.Effect<boolean>,
    ) => Effect.Effect<boolean>;
    readonly tunnelConnected: Stream.Stream<void>;
    readonly withLinkStateLock: <A, E, R>(effect: Effect.Effect<A, E, R>) => Effect.Effect<A, E, R>;
  }
>()("t3/cloud/ManagedEndpointRuntime/CloudManagedEndpointRuntime") {}

interface ActiveConnector {
  readonly child: ChildProcessSpawner.ChildProcessHandle;
  readonly executable: RelayClient.AvailableRelayClient;
  readonly scope: Scope.Closeable;
  readonly configKey: string;
  readonly config: RelayManagedEndpointRuntimeConfig;
  readonly startedAtMillis: number;
  readonly registered: Deferred.Deferred<void>;
  readonly metricsAddress: Deferred.Deferred<string>;
  readonly protocol: Ref.Ref<string | null>;
}

// An instantly failing relay client otherwise respawns ~100 times/sec and exhausts V8 via tracing.
const isRotatingFileSinkError = Schema.is(RotatingFileSinkError);

const RELAY_RESTART_STABLE_UPTIME_MS = 30_000;
const RELAY_RESTART_BACKOFF_BASE_MS = 1_000;
const RELAY_RESTART_BACKOFF_MAX_MS = 60_000;
// Newly created tunnels can fail authorization briefly while Cloudflare propagates their token.
const TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY = 4;
// Unknown rejection wording can leave a dead tunnel unconnected; periodically request recovery.
const CONNECTOR_REGISTRATION_TIMEOUT = Duration.minutes(3);
// cloudflared resolves edges only at startup and can keep dialing stale addresses after a network change.
const CONNECTOR_READINESS_INTERVAL = Duration.seconds(30);
const CONNECTOR_READINESS_REQUEST_TIMEOUT = Duration.seconds(5);
const CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART = 6;
const CONNECTOR_NOT_READY_CHECKS_MAX = 60;
const RELAY_CLIENT_METRICS_LINE = /\bStarting metrics server on (127\.0\.0\.1:\d+)\/metrics\b/u;
const RELAY_CLIENT_PROTOCOL = /\bprotocol=(\w+)/u;
// cloudflared's metrics server: `/ready` answers 200 or 503 with this body, and
// `/config` holds the ingress rules the edge pushed after the first connection.
const ConnectorReadyBody = Schema.Struct({ readyConnections: Schema.Number });
const ConnectorConfigBody = Schema.Struct({
  config: Schema.Struct({
    ingress: Schema.optional(
      Schema.NullOr(Schema.Array(Schema.Struct({ hostname: Schema.optional(Schema.String) }))),
    ),
  }),
});
// Registration resets cloudflared's own fallback, even when QUIC stalls again
// immediately. Keep that evidence across registrations and connector restarts.
const QUIC_STALLS_BEFORE_HTTP2 = 30;
const QUIC_STALL_WINDOW_MS = 5 * 60_000;
// Retry QUIC on a later startup after transient failures or network changes; reuse never renews this hint.
const HTTP2_FALLBACK_TTL_MS = 24 * 60 * 60_000;
const HTTP2_FALLBACK_STATE = "cloud-endpoint-http2-fallback";
const Http2Fallback = Schema.fromJsonString(Schema.Struct({ fellBackAt: Schema.Number }));
const decodeHttp2Fallback = Schema.decodeUnknownOption(Http2Fallback);
const encodeHttp2Fallback = Schema.encodeSync(Http2Fallback);
// A failed background install of the pinned relay client is retried no sooner
// than this, so a crash-looping connector or an offline host does not redownload
// on every reconcile.
const RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS = 10 * 60_000;
export const RELAY_CLIENT_LOG_FILE = "relay-client.ndjson";

/** Preserve user-chosen PATH binaries and overrides; CLI downloads require consent. */
function needsPinnedRelayClient(executable: RelayClient.RelayClientStatus): boolean {
  return (
    executable.status === "missing" ||
    (executable.status === "available" &&
      executable.source === "managed" &&
      !RelayClient.isPinnedManagedRelayClient(executable))
  );
}

export function classifyRelayClientOutput(
  line: string,
): "connected" | "unregistered" | "warning" | "debug" {
  if (/\bRegistered tunnel connection\b/iu.test(line)) {
    return "connected";
  }
  // cloudflared uses zerolog FTL/PNC for fatal/panic; classify them above debug too.
  if (/\b(?:ERR|WRN|FTL|PNC)\b/u.test(line)) {
    return "warning";
  }
  return /\bINF\b.*\bUnregistered tunnel connection\b/u.test(line) ? "unregistered" : "debug";
}

/** Edge rejection wording varies; any Unauthorized registration error counts. */
export function isRejectedRelayClientTunnelOutput(line: string): boolean {
  return (
    /\bRegister tunnel error from server side\b/iu.test(line) &&
    /error="(?:Unauthorized:[^"]*|Failed to get tunnel|Tunnel not found|Record for tunnel not found|Invalid tunnel secret)"/iu.test(
      line,
    )
  );
}

export function isRetryableManagedEndpointRuntimeStatus(status: unknown): boolean {
  if (typeof status !== "object" || status === null || !("status" in status)) {
    return false;
  }
  if (status.status !== "failed" || !("failure" in status)) {
    return false;
  }
  return status.failure === "not-installed" || status.failure === "spawn-failed";
}

function runtimeConfigKey(config: RelayManagedEndpointRuntimeConfig): string {
  return JSON.stringify({
    providerKind: config.providerKind,
    connectorToken: config.connectorToken,
    tunnelId: config.tunnelId ?? null,
    tunnelName: config.tunnelName ?? null,
  });
}

const stopConnector = (connector: ActiveConnector | null) =>
  connector
    ? Scope.close(connector.scope, Exit.void).pipe(
        Effect.tap(() =>
          Effect.logInfo("Relay client stopped", {
            pid: Number(connector.child.pid),
          }),
        ),
        Effect.ignore,
      )
    : Effect.void;

/** @public */
export const make = Effect.gen(function* () {
  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
  const relayClient = yield* RelayClient.RelayClient;
  const secrets = yield* ServerSecretStore.ServerSecretStore;
  const httpClient = HttpClient.withScope(yield* HttpClient.HttpClient);
  // Without T3 Connect no connector ever runs, whoever asks for one.
  const connectAllowed = ManagedLimits.allowsConnect(yield* ManagedPolicy.ManagedPolicy);
  const activeRef = yield* Ref.make<ActiveConnector | null>(null);
  const desiredConfigRef = yield* Ref.make<RelayManagedEndpointRuntimeConfig | null>(null);
  const recoveryRequests = yield* Queue.sliding<RelayManagedEndpointRuntimeConfig>(1);
  const tunnelConnections = yield* Queue.sliding<void>(1);
  const readinessChanges = yield* Queue.sliding<void>(1);
  const reconcileSemaphore = yield* Semaphore.make(1);
  const restartDelayRef = yield* Ref.make(0);
  const linkStateSemaphore = yield* Semaphore.make(1);
  const runtimeScope = yield* Effect.scope;
  const installInFlightRef = yield* Ref.make(false);
  const lastInstallFailureAtRef = yield* Ref.make<number | null>(null);
  const prunedRef = yield* Ref.make(false);
  const retryLoopRunningRef = yield* Ref.make(false);
  const notReadyChecksBeforeRestartRef = yield* Ref.make(CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
  // An explicit cloudflared transport in the environment is the user's choice.
  const env = yield* HostProcess.Environment;
  const transportChosenByUser = Option.isSome(
    yield* Config.String("TUNNEL_TRANSPORT_PROTOCOL").pipe(Config.option),
  );
  const previousFallback = transportChosenByUser
    ? Option.none()
    : yield* secrets.get(HTTP2_FALLBACK_STATE).pipe(
        Effect.map(Option.flatMap((bytes) => decodeHttp2Fallback(new TextDecoder().decode(bytes)))),
        Effect.orElseSucceed(() => Option.none()),
      );
  const now = yield* Clock.currentTimeMillis;
  const http2Ref = yield* Ref.make(
    Option.isSome(previousFallback) &&
      now >= previousFallback.value.fellBackAt &&
      now - previousFallback.value.fellBackAt < HTTP2_FALLBACK_TTL_MS,
  );
  const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]);
  let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];

  const { logsDir } = yield* ServerConfig.ServerConfig;
  const path = yield* Path.Path;
  const relayClientLogPath = path.join(logsDir, RELAY_CLIENT_LOG_FILE);
  // Synchronous appends avoid a flush timer for this low-volume log.
  const fallbackLoggers = yield* Effect.service(Logger.CurrentLoggers);
  let writeFailureReported = false;
  const relayClientLog = yield* Effect.try({
    try: () =>
      new RotatingFileSink({
        filePath: relayClientLogPath,
        maxBytes: 1024 * 1024,
        maxFiles: 2,
        throwOnError: true,
      }),
    catch: (cause) =>
      isRotatingFileSinkError(cause)
        ? cause
        : new RotatingFileSinkError({
            operation: "initialize",
            filePath: relayClientLogPath,
            cause,
          }),
  }).pipe(
    Effect.map((sink) =>
      Logger.make((options) => {
        try {
          const entry = Logger.formatJson.log(options);
          const bytes = Buffer.byteLength(entry) + 1;
          sink.write(
            bytes <= 1024 * 1024
              ? `${entry}\n`
              : `${JSON.stringify({
                  timestamp: options.date.toISOString(),
                  level: options.logLevel.toUpperCase(),
                  message: ["Relay client log entry omitted: exceeds 1 MiB", { bytes }],
                })}\n`,
          );
        } catch {
          if (!writeFailureReported) {
            writeFailureReported = true;
            for (const logger of fallbackLoggers) {
              logger.log({
                ...options,
                logLevel: "Warn",
                message: ["Relay client log unavailable", { filePath: relayClientLogPath }],
              });
            }
          }
        }
      }),
    ),
    Effect.catch((error) =>
      Effect.logWarning("Relay client log unavailable", {
        filePath: relayClientLogPath,
        cause: error,
      }).pipe(Effect.as(undefined)),
    ),
  );
  // Forked connector fibers inherit this logger after the applyConfig span ends.
  const withRelayClientLog = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
    relayClientLog ? Effect.withLogger(effect, relayClientLog) : effect;

  const stopActive = Effect.gen(function* () {
    const active = yield* Ref.getAndSet(activeRef, null);
    yield* stopConnector(active);
  });

  // Lives until connector exit and starts replacements; avoid nested lifetime spans.
  const superviseConnector = Effect.fnUntraced(
    function* (connector: ActiveConnector) {
      const result = yield* Effect.result(connector.child.exitCode);
      const activeAtExit = yield* Ref.get(activeRef);
      if (
        activeAtExit?.child.pid !== connector.child.pid ||
        activeAtExit.configKey !== connector.configKey
      ) {
        return;
      }
      const uptimeMillis = (yield* Clock.currentTimeMillis) - connector.startedAtMillis;
      // Back off outside the semaphore so config changes remain responsive; recheck config afterwards.
      const restartDelayMillis = yield* Ref.modify(restartDelayRef, (current) => {
        if (uptimeMillis >= RELAY_RESTART_STABLE_UPTIME_MS) {
          return [0, 0];
        }
        return [
          current,
          current === 0
            ? RELAY_RESTART_BACKOFF_BASE_MS
            : Math.min(current * 2, RELAY_RESTART_BACKOFF_MAX_MS),
        ];
      });
      if (restartDelayMillis > 0) {
        yield* Effect.logWarning("Relay client is crash-looping; delaying restart", {
          pid: Number(connector.child.pid),
          uptimeMillis,
          restartDelayMillis,
          tunnelId: connector.config.tunnelId,
          tunnelName: connector.config.tunnelName,
        });
        yield* Effect.sleep(Duration.millis(restartDelayMillis));
      }
      yield* reconcileSemaphore.withPermits(1)(
        Effect.gen(function* () {
          const active = yield* Ref.get(activeRef);
          if (
            active?.child.pid !== connector.child.pid ||
            active.configKey !== connector.configKey
          ) {
            return;
          }
          yield* Ref.set(activeRef, null);
          yield* stopConnector(connector);

          const desiredConfig = yield* Ref.get(desiredConfigRef);
          if (
            !desiredConfig ||
            desiredConfig.providerKind !== "cloudflare_tunnel" ||
            runtimeConfigKey(desiredConfig) !== connector.configKey
          ) {
            return;
          }

          yield* Effect.logWarning("Relay client exited; restarting", {
            pid: Number(connector.child.pid),
            ...(Result.isSuccess(result)
              ? { exitCode: Number(result.success) }
              : { cause: result.failure }),
            tunnelId: connector.config.tunnelId,
            tunnelName: connector.config.tunnelName,
          });
          yield* Queue.offer(recoveryRequests, connector.config);
          yield* reconcileConfig(desiredConfig);
        }),
      );
    },
    Effect.catchCause((cause) => Effect.logWarning("Relay client supervisor failed", { cause })),
  );

  // Keep the old connector until its replacement registers, preserving the relay connection.
  const installPinnedRelayClient = Effect.gen(function* () {
    if (yield* Ref.getAndSet(installInFlightRef, true)) return;
    yield* Effect.logInfo("Installing the pinned relay client", {
      version: RelayClient.CLOUDFLARED_VERSION,
    });
    yield* relayClient.install.pipe(
      Effect.tap(() => Ref.set(lastInstallFailureAtRef, null)),
      Effect.flatMap((installed) =>
        reconcileSemaphore.withPermits(1)(
          Effect.gen(function* () {
            const desiredConfig = yield* Ref.get(desiredConfigRef);
            if (!desiredConfig || desiredConfig.providerKind !== "cloudflare_tunnel") return;
            const active = yield* Ref.get(activeRef);
            // Self-updated binaries share the pinned path; compare versions to detect replacements.
            if (
              active?.executable.executablePath === installed.executablePath &&
              active.executable.version === installed.version
            ) {
              return;
            }
            // A scanner can block a fresh binary's probe; retry instead of restarting onto the older binary.
            const resolved = yield* relayClient.resolve;
            if (
              resolved.status !== "available" ||
              resolved.executablePath !== installed.executablePath ||
              resolved.version !== installed.version
            ) {
              return;
            }
            yield* Effect.logInfo("Relay client installed; moving the connector to it", {
              version: installed.version,
              previousVersion: active?.executable.version,
            });
            const previous = yield* Ref.getAndSet(activeRef, null);
            const status = yield* reconcileConfig(desiredConfig);
            const next = yield* Ref.get(activeRef);
            if (!next) {
              yield* Effect.logWarning("Relay client did not start after the update", status);
              // The supervisor skips exits while detached; request recovery explicitly.
              const previousRunning = previous
                ? yield* previous.child.isRunning.pipe(Effect.orElseSucceed(() => false))
                : false;
              if (previous && previousRunning) {
                yield* Ref.set(activeRef, previous);
              } else {
                yield* stopConnector(previous);
                yield* Queue.offer(recoveryRequests, desiredConfig);
              }
              return;
            }
            if (previous) {
              yield* Deferred.await(next.registered).pipe(
                Effect.ensuring(stopConnector(previous)),
                Effect.forkIn(next.scope),
              );
            }
          }),
        ),
      ),
      Effect.catchTags({
        RelayClientInstallError: (error) =>
          Clock.currentTimeMillis.pipe(
            Effect.flatMap((failedAt) => Ref.set(lastInstallFailureAtRef, failedAt)),
            Effect.andThen(
              Effect.logWarning("Could not install the pinned relay client", {
                reason: error.reason,
                message: error.message,
                cause: error.cause,
              }),
            ),
          ),
      }),
      Effect.ensuring(Ref.set(installInFlightRef, false)),
    );
  });

  const ensurePinnedRelayClient = Effect.gen(function* () {
    const now = yield* Clock.currentTimeMillis;
    const lastFailureAt = yield* Ref.get(lastInstallFailureAtRef);
    if (lastFailureAt !== null && now - lastFailureAt < RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS) {
      return;
    }
    yield* installPinnedRelayClient;
  }).pipe(Effect.forkIn(runtimeScope), Effect.asVoid);

  // Retry installs skipped by applyConfig without the failure gate.
  const retryPinnedRelayClientInstall = Effect.gen(function* () {
    while (true) {
      yield* Effect.sleep(Duration.millis(RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS));
      const desiredConfig = yield* Ref.get(desiredConfigRef);
      if (!desiredConfig || desiredConfig.providerKind !== "cloudflare_tunnel") return;
      // An installed pin still needs activation after a failed probe or while an older connector runs.
      const active = yield* Ref.get(activeRef);
      if (active && !needsPinnedRelayClient(active.executable)) return;
      yield* installPinnedRelayClient;
    }
  }).pipe(Effect.ensuring(Ref.set(retryLoopRunningRef, false)));

  const startPinnedRelayClientRetries = Effect.gen(function* () {
    if (yield* Ref.getAndSet(retryLoopRunningRef, true)) return;
    yield* Effect.forkIn(retryPinnedRelayClientInstall, runtimeScope);
  });

  // Registration retries can last for the connector lifetime; keep this loop untraced.
  const watchConnectorRegistration = Effect.fnUntraced(
    function* (connector: ActiveConnector) {
      yield* Effect.sleep(CONNECTOR_REGISTRATION_TIMEOUT);
      if (yield* Deferred.isDone(connector.registered)) return true;
      yield* Effect.logWarning(
        "Relay client has not registered a tunnel connection; requesting recovery",
        {
          pid: Number(connector.child.pid),
          tunnelId: connector.config.tunnelId,
          tunnelName: connector.config.tunnelName,
        },
      );
      yield* Queue.offer(recoveryRequests, connector.config);
      return false;
    },
    Effect.repeat({ until: (connected) => connected }),
  );

  // Only an explicit 503 counts: a failed request says nothing about the
  // Readiness polling runs for the connector lifetime; keep this loop untraced.
  const watchConnectorReadiness = Effect.fnUntraced(function* (connector: ActiveConnector) {
    const readyUrl = `http://${yield* Deferred.await(connector.metricsAddress)}/ready`;
    let notReadyChecks = 0;
    let lastReady: boolean | null = null;
    while (true) {
      yield* Effect.sleep(CONNECTOR_READINESS_INTERVAL);
      const status = yield* httpClient.get(readyUrl).pipe(
        Effect.map((response) => response.status),
        Effect.scoped,
        Effect.withTracerEnabled(false),
        Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT),
        Effect.orElseSucceed(() => Option.none<number>()),
      );
      if (Option.isNone(status)) continue;
      if (status.value === 200 || status.value === 503) {
        const ready = status.value === 200;
        if (ready !== lastReady) yield* Queue.offer(readinessChanges, undefined);
        lastReady = ready;
      }
      if (status.value === 200) {
        notReadyChecks = 0;
        yield* Ref.set(notReadyChecksBeforeRestartRef, CONNECTOR_NOT_READY_CHECKS_BEFORE_RESTART);
        continue;
      }
      if (status.value !== 503) continue;
      notReadyChecks += 1;
      const threshold = yield* Ref.get(notReadyChecksBeforeRestartRef);
      if (notReadyChecks < threshold) continue;
      notReadyChecks = 0;
      yield* Ref.set(
        notReadyChecksBeforeRestartRef,
        Math.min(threshold * 2, CONNECTOR_NOT_READY_CHECKS_MAX),
      );
      yield* Effect.logWarning("Relay client has no tunnel connection; restarting it", {
        pid: Number(connector.child.pid),
        notReadyChecks: threshold,
        tunnelId: connector.config.tunnelId,
        tunnelName: connector.config.tunnelName,
      });
      yield* connector.child.kill().pipe(Effect.ignore);
    }
  });

  const switchToHttp2 = Effect.fnUntraced(function* (connector: ActiveConnector) {
    const active = yield* Ref.get(activeRef);
    const desiredConfig = yield* Ref.get(desiredConfigRef);
    if ((yield* Ref.get(http2Ref)) || active !== connector || !desiredConfig) return;
    yield* Ref.set(http2Ref, true);
    yield* secrets
      .set(
        HTTP2_FALLBACK_STATE,
        new TextEncoder().encode(
          encodeHttp2Fallback({ fellBackAt: yield* Clock.currentTimeMillis }),
        ),
      )
      .pipe(
        Effect.catch(() => Effect.logWarning("Could not persist the relay client HTTP/2 fallback")),
      );
    yield* Effect.logWarning("Relay client keeps losing QUIC connections; switching to HTTP/2", {
      pid: Number(connector.child.pid),
      tunnelId: connector.config.tunnelId,
      tunnelName: connector.config.tunnelName,
    });
    // Shutdown must not interrupt cleanup after active ownership is cleared.
    yield* Effect.uninterruptible(stopActive);
    const status = yield* reconcileConfig(desiredConfig);
    // A failed spawn has no supervisor to request recovery.
    if (status.status !== "running") {
      yield* Queue.offer(recoveryRequests, desiredConfig);
    }
  }, reconcileSemaphore.withPermits(1));

  const noteQuicStall = Effect.fnUntraced(function* (connector: ActiveConnector) {
    if (transportChosenByUser || (yield* Ref.get(http2Ref))) return;
    const now = yield* Clock.currentTimeMillis;
    if ((yield* Ref.get(activeRef)) !== connector) return;
    const stalls = yield* Ref.updateAndGet(quicStallsRef, (previous) => [
      ...previous.filter((at) => now - at < QUIC_STALL_WINDOW_MS),
      now,
    ]);
    if (stalls.length < QUIC_STALLS_BEFORE_HTTP2) return;
    // Stopping the connector interrupts the output observer that called this.
    yield* Effect.forkIn(switchToHttp2(connector), runtimeScope);
  });

  const observeConnectorOutput = (connector: ActiveConnector) => {
    let rejectedRegistrations = 0;

    return connector.child.all.pipe(
      Stream.decodeText(),
      Stream.splitLines,
      Stream.map((line) => line.trim()),
      Stream.filter((line) => line.length > 0),
      Stream.runForEach((line) => {
        const output = line.replaceAll(connector.config.connectorToken, "<redacted>");
        const attributes = {
          pid: Number(connector.child.pid),
          tunnelId: connector.config.tunnelId,
          tunnelName: connector.config.tunnelName,
          output,
        };
        switch (classifyRelayClientOutput(line)) {
          case "connected":
            rejectedRegistrations = 0;
            return Deferred.succeed(connector.registered, undefined).pipe(
              Effect.andThen(
                Ref.set(connector.protocol, RELAY_CLIENT_PROTOCOL.exec(line)?.[1] ?? null),
              ),
              Effect.andThen(
                Effect.logInfo("Relay client tunnel connection registered", attributes),
              ),
              Effect.andThen(Queue.offer(tunnelConnections, undefined)),
              Effect.andThen(Queue.offer(readinessChanges, undefined)),
              Effect.andThen(
                RelayClient.isPinnedManagedRelayClient(connector.executable)
                  ? Ref.getAndSet(prunedRef, true).pipe(
                      Effect.flatMap((pruned) =>
                        pruned ? Effect.void : relayClient.pruneManagedVersions,
                      ),
                    )
                  : Effect.void,
              ),
              Effect.asVoid,
            );
          case "unregistered":
            return Effect.logInfo("Relay client tunnel connection unregistered", attributes);
          case "warning":
            if (/\btimeout: no recent network activity\b/u.test(line)) {
              return noteQuicStall(connector).pipe(
                Effect.andThen(
                  Effect.logWarning("Relay client reported a transport warning", attributes),
                ),
              );
            }
            if (isRejectedRelayClientTunnelOutput(line)) {
              rejectedRegistrations += 1;
              if (rejectedRegistrations >= TUNNEL_AUTHORIZATION_FAILURES_BEFORE_RECOVERY) {
                rejectedRegistrations = 0;
                return Effect.logWarning(
                  "Relay client tunnel was rejected; requesting recovery",
                  attributes,
                ).pipe(
                  Effect.andThen(Queue.offer(recoveryRequests, connector.config)),
                  Effect.asVoid,
                );
              }
            }
            return Effect.logWarning("Relay client reported a transport warning", attributes);
          case "debug": {
            const metricsAddress = RELAY_CLIENT_METRICS_LINE.exec(line)?.[1];
            return metricsAddress
              ? Deferred.succeed(connector.metricsAddress, metricsAddress).pipe(
                  Effect.andThen(Effect.logDebug("Relay client output", attributes)),
                )
              : Effect.logDebug("Relay client output", attributes);
          }
        }
      }),
      Effect.catchCause((cause) =>
        Effect.logWarning("Relay client output observer failed", {
          cause,
          pid: Number(connector.child.pid),
          tunnelId: connector.config.tunnelId,
          tunnelName: connector.config.tunnelName,
        }),
      ),
    );
  };

  reconcileConfig = Effect.fn("CloudManagedEndpointRuntime.reconcileConfig")(function* (config) {
    if (!config || config.providerKind !== "cloudflare_tunnel") {
      yield* stopActive;
      return config
        ? { status: "unsupported", providerKind: config.providerKind }
        : { status: "disabled" };
    }

    const nextConfigKey = runtimeConfigKey(config);
    const active = yield* Ref.get(activeRef);
    if (active?.configKey === nextConfigKey) {
      const isRunning = yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false));
      if (isRunning) {
        return {
          status: "running",
          providerKind: "cloudflare_tunnel",
          pid: Number(active.child.pid),
          ...(active.config.tunnelId ? { tunnelId: active.config.tunnelId } : {}),
          ...(active.config.tunnelName ? { tunnelName: active.config.tunnelName } : {}),
        } satisfies CloudManagedEndpointRuntimeStatus;
      }
    }

    yield* stopActive;

    const executable = yield* relayClient.resolve;
    if (needsPinnedRelayClient(executable)) {
      yield* ensurePinnedRelayClient;
      yield* startPinnedRelayClientRetries;
    }
    if (executable.status !== "available") {
      return {
        status: "failed",
        providerKind: "cloudflare_tunnel",
        failure: executable.status === "unsupported" ? "unsupported-platform" : "not-installed",
        reason:
          executable.status === "unsupported"
            ? `Relay client is unsupported on ${executable.platform}-${executable.arch}.`
            : "The relay client is not installed.",
        ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
        ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
      } satisfies CloudManagedEndpointRuntimeStatus;
    }

    const http2 = yield* Ref.get(http2Ref);
    const connectorScope = yield* Scope.make("sequential");
    const child = yield* spawner
      .spawn(
        ChildProcess.make(
          executable.executablePath,
          [
            "tunnel",
            "--no-autoupdate",
            "--loglevel",
            "info",
            "--output",
            "default",
            "--metrics",
            "127.0.0.1:0",
            ...(http2 ? ["--protocol", "http2"] : []),
            "run",
          ],
          {
            detached: false,
            env: {
              ...env,
              TUNNEL_TOKEN: config.connectorToken,
            },
            shell: false,
            stderr: "pipe",
            stdout: "pipe",
          },
        ),
      )
      .pipe(
        Effect.provideService(Scope.Scope, connectorScope),
        Effect.tap((child) =>
          Effect.logInfo("Relay client process started; waiting for tunnel connection", {
            pid: Number(child.pid),
            source: executable.source,
            version: executable.version,
            protocol: http2 ? "http2" : "auto",
            tunnelId: config.tunnelId,
            tunnelName: config.tunnelName,
          }),
        ),
        Effect.catch((cause) =>
          Effect.logWarning("Failed to start relay client", {
            cause,
            tunnelId: config.tunnelId,
            tunnelName: config.tunnelName,
          }).pipe(
            Effect.andThen(Scope.close(connectorScope, Exit.void).pipe(Effect.ignore)),
            Effect.as({
              status: "failed",
              providerKind: "cloudflare_tunnel",
              failure: "spawn-failed",
              reason: String(cause),
              ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
              ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
            } satisfies CloudManagedEndpointRuntimeStatus),
          ),
        ),
      );

    if ("status" in child && child.status === "failed") {
      return child;
    }

    if (!("status" in child)) {
      const connector = {
        child,
        executable,
        scope: connectorScope,
        configKey: nextConfigKey,
        config,
        startedAtMillis: yield* Clock.currentTimeMillis,
        registered: yield* Deferred.make<void>(),
        metricsAddress: yield* Deferred.make<string>(),
        protocol: yield* Ref.make<string | null>(null),
      } satisfies ActiveConnector;
      yield* Ref.set(activeRef, connector);
      yield* Effect.forkIn(observeConnectorOutput(connector), connectorScope);
      yield* Effect.forkIn(superviseConnector(connector), connectorScope);
      yield* Effect.forkIn(watchConnectorRegistration(connector), connectorScope);
      yield* Effect.forkIn(watchConnectorReadiness(connector), connectorScope);
      return {
        status: "running",
        providerKind: "cloudflare_tunnel",
        pid: Number(child.pid),
        ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
        ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
      } satisfies CloudManagedEndpointRuntimeStatus;
    }

    return {
      status: "failed",
      providerKind: "cloudflare_tunnel",
      failure: "spawn-failed",
      reason: "Relay client did not start.",
      ...(config.tunnelId ? { tunnelId: config.tunnelId } : {}),
      ...(config.tunnelName ? { tunnelName: config.tunnelName } : {}),
    } satisfies CloudManagedEndpointRuntimeStatus;
  });

  const applyConfig = Effect.fn("CloudManagedEndpointRuntime.applyConfig")(
    (requested: RelayManagedEndpointRuntimeConfig | null) =>
      reconcileSemaphore.withPermits(1)(
        Effect.gen(function* () {
          const config = connectAllowed ? requested : null;
          // Keep backoff when recovery returns the same tunnel/token, or every recovery respawns a crash loop.
          const desired = yield* Ref.get(desiredConfigRef);
          const unchanged =
            desired !== null &&
            config !== null &&
            runtimeConfigKey(desired) === runtimeConfigKey(config);
          if (!unchanged) {
            yield* Ref.set(restartDelayRef, 0);
          }
          yield* Ref.set(desiredConfigRef, config);
          return yield* reconcileConfig(config);
        }),
      ),
    withRelayClientLog,
  );

  const restartConnector = Effect.fn("CloudManagedEndpointRuntime.restartConnector")(function* (
    pid: number,
    stillDead: Effect.Effect<boolean>,
  ) {
    const active = yield* Ref.get(activeRef);
    if (!active || Number(active.child.pid) !== pid || !(yield* stillDead)) return false;
    yield* Effect.logWarning(
      "Cloudflare has no live connection from the relay client; restarting it",
      { pid, tunnelId: active.config.tunnelId, tunnelName: active.config.tunnelName },
    );
    // The supervisor starts the replacement with its usual crash-loop backoff.
    return yield* active.child.kill().pipe(
      Effect.as(true),
      Effect.orElseSucceed(() => false),
    );
  }, reconcileSemaphore.withPermits(1));

  const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>
    httpClient.get(url).pipe(
      Effect.flatMap(HttpClientResponse.schemaBodyJson(schema)),
      Effect.scoped,
      Effect.withTracerEnabled(false),
      Effect.timeoutOption(CONNECTOR_READINESS_REQUEST_TIMEOUT),
      Effect.map(Option.getOrNull),
      Effect.orElseSucceed(() => null),
    );

  const inspectConnector = Effect.gen(function* () {
    const active = yield* Ref.get(activeRef);
    if (!active || !(yield* active.child.isRunning.pipe(Effect.orElseSucceed(() => false)))) {
      return null;
    }
    const pid = Number(active.child.pid);
    const protocol = yield* Ref.get(active.protocol);
    if (!(yield* Deferred.isDone(active.metricsAddress))) {
      return { pid, readyConnections: null, hostname: null, protocol } satisfies ConnectorFacts;
    }
    const metricsUrl = `http://${yield* Deferred.await(active.metricsAddress)}`;
    const [ready, config] = yield* Effect.all(
      [
        readMetrics(`${metricsUrl}/ready`, ConnectorReadyBody),
        readMetrics(`${metricsUrl}/config`, ConnectorConfigBody),
      ],
      { concurrency: 2 },
    );
    return {
      pid,
      readyConnections: ready?.readyConnections ?? null,
      hostname: config?.config.ingress?.find((rule) => rule.hostname)?.hostname ?? null,
      protocol,
    } satisfies ConnectorFacts;
  });

  const runtime = CloudManagedEndpointRuntime.of({
    applyConfig,
    inspectConnector,
    readinessChanged: Stream.fromQueue(readinessChanges),
    recoveryRequests: Stream.fromQueue(recoveryRequests),
    requestRecovery: Effect.fn("CloudManagedEndpointRuntime.requestRecovery")(function (config) {
      return Queue.offer(recoveryRequests, config).pipe(Effect.asVoid);
    }),
    restartConnector,
    tunnelConnected: Stream.fromQueue(tunnelConnections),
    withLinkStateLock: linkStateSemaphore.withPermits(1),
  });

  yield* Effect.addFinalizer(() => runtime.applyConfig(null));
  return runtime;
});

export const layer = Layer.effect(CloudManagedEndpointRuntime, make);