sailfork / t3code-github / apps/ server/ src/ cloud/ ConnectReachability.ts

import * as NodeNet from "node:net";

import {
  connectReachabilityHelpUrl,
  type EnvironmentConnectReachability,
  formatConnectReachable,
} from "@t3tools/contracts";
import { readTailscaleExitNode, type TailscaleExitNode } from "@t3tools/tailscale";
import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Ref from "effect/Ref";
import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
import * as SynchronizedRef from "effect/SynchronizedRef";
import { HttpClient } from "effect/http";
import { ChildProcessSpawner } from "effect/process";

import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
import { CLOUD_ENDPOINT_RUNTIME_CONFIG } from "./config.ts";
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";

// cloudflared finds the edge through the SRV record `_v2-origintunneld._tcp.argotunnel.com`,
// which names these two hosts on port 7844. QUIC and HTTP/2 both use that port.
const TUNNEL_EDGE_HOSTS = ["region1.v2.argotunnel.com", "region2.v2.argotunnel.com"];
const TUNNEL_EDGE_PORT = 7844;
const EDGE_CONNECT_TIMEOUT = Duration.seconds(5);
const ENDPOINT_REQUEST_TIMEOUT = Duration.seconds(10);
const ERROR_BODY_READ_TIMEOUT = Duration.seconds(2);
// cloudflared registers several connections within a few seconds; they are one change.
const READINESS_CHANGE_DEBOUNCE = Duration.seconds(3);
// Readiness changes miss failures and recoveries past cloudflared (DNS, origin, edge errors).
const RECHECK_INTERVAL = Duration.minutes(5);
// cloudflared can keep reporting ready connections the edge already dropped, so its
// own readiness watch never restarts it. A 530 can also be brief, so restart only once it
// lasts, and wait longer after each restart that did not help.
const DEAD_CONNECTOR_RESTART_AFTER = Duration.minutes(2);
const DEAD_CONNECTOR_RESTART_AFTER_MAX = Duration.hours(1);

export type EndpointProbe =
  | { readonly kind: "status"; readonly status: number; readonly cloudflareError?: string }
  | { readonly kind: "dns_failed" | "timeout" | "network_error" };

/** Everything one check learned. `null` means that step did not run. */
export interface ReachabilityFacts {
  readonly managedTunnelConfigured: boolean;
  readonly connector: ManagedEndpointRuntime.ConnectorFacts | null;
  readonly endpoint: EndpointProbe | null;
  readonly edgeReachable: boolean | null;
  readonly exitNode: TailscaleExitNode | null;
}

type Unreachable = Extract<EnvironmentConnectReachability, { status: "unreachable" }>;

function exitNodeHint(exitNode: TailscaleExitNode | null): string | null {
  if (!exitNode) return null;
  const name = exitNode.name ? ` "${exitNode.name}"` : "";
  return `Tailscale exit node${name} is on. Traffic to Cloudflare goes through it.`;
}

/** Turns what one check learned into the answer every surface shows. */
export function classifyReachability(
  facts: ReachabilityFacts,
  checkedAt: string,
): EnvironmentConnectReachability {
  const { connector, endpoint, exitNode } = facts;
  const fail = (
    cause: Unreachable["cause"],
    reason: string,
    fix: string,
    hint: string | null = null,
  ): Unreachable => ({ status: "unreachable", checkedAt, cause, reason, hint, fix });
  const restart = "Restart T3 Code. If it stays, run `t3 connect` again.";

  if (!facts.managedTunnelConfigured) {
    // A running server reads the link `t3 connect` stores only at startup.
    return fail(
      "not_linked",
      "The server has not set up a T3 Connect tunnel.",
      "Run `t3 connect`, then restart T3 Code so it applies the link.",
    );
  }
  if (connector === null) {
    return fail(
      "connector_not_running",
      "The tunnel client (cloudflared) is not running.",
      "Restart T3 Code. If it stays down, check the server log for relay client errors.",
    );
  }
  const status = endpoint?.kind === "status" ? endpoint.status : null;
  if (status === 200) {
    return {
      status: "reachable",
      checkedAt,
      readyConnections: connector.readyConnections,
      protocol: connector.protocol,
    };
  }
  // The edge test is TCP only. With QUIC connections up, a 502 from a working tunnel, or
  // DNS already failing, a failed TCP test would hide the real cause.
  if (
    facts.edgeReachable === false &&
    status !== 502 &&
    !(connector.readyConnections && connector.readyConnections > 0) &&
    endpoint?.kind !== "dns_failed"
  ) {
    return fail(
      "edge_unreachable",
      `Cloudflare's tunnel port (${TUNNEL_EDGE_PORT}) times out from this machine.`,
      exitNode
        ? `tailscale set --exit-node=   (or allow outbound ${TUNNEL_EDGE_PORT} on that network)`
        : `Allow outbound TCP and UDP port ${TUNNEL_EDGE_PORT}, or turn off the VPN or firewall that blocks it.`,
      exitNodeHint(exitNode) ?? "A VPN, proxy, or firewall may block it.",
    );
  }
  if (status === 502) {
    return fail(
      "origin_unreachable",
      "The tunnel is up, but it cannot reach the T3 server on this machine.",
      "Restart T3 Code so the tunnel uses the port the server listens on.",
    );
  }
  if (endpoint === null || connector.readyConnections === 0) {
    return fail(
      "connector_not_ready",
      "cloudflared has not connected to Cloudflare yet.",
      "Wait a minute. If it stays, check the network and the server log.",
    );
  }
  const cloudflareError = endpoint.kind === "status" ? endpoint.cloudflareError : undefined;
  // Cloudflare answers 530 with error 1033 when the tunnel has no live connector. Other
  // codes, such as 1016 for a broken DNS record, are not fixed by restarting cloudflared.
  if (status === 530 && (cloudflareError === undefined || cloudflareError === "1033")) {
    return fail(
      "no_connector",
      "Cloudflare has no live connection from this machine for the tunnel.",
      restart,
    );
  }
  if (status !== null) {
    const code = cloudflareError === undefined ? "" : ` (Cloudflare error ${cloudflareError})`;
    return fail(
      "unexpected_response",
      `The T3 Connect address answered with HTTP ${status}${code}.`,
      restart,
    );
  }
  return endpoint.kind === "dns_failed"
    ? fail(
        "dns_failed",
        "The T3 Connect address does not resolve from this machine.",
        "Check the DNS settings on this machine or its VPN.",
        exitNodeHint(exitNode),
      )
    : fail(
        "timeout",
        "The T3 Connect address did not answer from this machine.",
        "Check the network, VPN, or proxy on this machine.",
        exitNodeHint(exitNode),
      );
}

/** The loud diagnosis the CLI prints and the server logs. */
export function formatReachabilityFailure(result: Unreachable): string {
  return [
    "✕ T3 Connect is NOT reachable from other devices",
    `  ${result.reason}`,
    ...(result.hint ? [`  ${result.hint}`] : []),
    `  Fix: ${result.fix}`,
    `  More info: ${connectReachabilityHelpUrl(result.cause)}`,
  ].join("\n");
}

function hasErrorCode(error: unknown, codes: ReadonlySet<string>): boolean {
  let current = error;
  for (let depth = 0; depth < 6 && typeof current === "object" && current !== null; depth++) {
    if ("code" in current && typeof current.code === "string" && codes.has(current.code)) {
      return true;
    }
    current = "cause" in current ? current.cause : undefined;
  }
  return false;
}

const DNS_ERROR_CODES = new Set(["ENOTFOUND", "EAI_AGAIN", "EAI_NONAME"]);

const canOpenTcp = (host: string, port: number) =>
  Effect.callback<boolean>((resume) => {
    const socket = NodeNet.connect({ host, port });
    const finish = (connected: boolean) => {
      socket.destroy();
      resume(Effect.succeed(connected));
    };
    socket.setTimeout(Duration.toMillis(EDGE_CONNECT_TIMEOUT), () => finish(false));
    socket.once("connect", () => finish(true));
    socket.once("error", () => finish(false));
    return Effect.sync(() => socket.destroy());
  });

export class ConnectReachability extends Context.Service<
  ConnectReachability,
  {
    /** Runs a check now. The server also runs one when the tunnel's readiness changes. */
    readonly check: Effect.Effect<EnvironmentConnectReachability>;
    readonly latest: Effect.Effect<Option.Option<EnvironmentConnectReachability>>;
  }
>()("t3/cloud/ConnectReachability") {}

const make = Effect.gen(function* () {
  const runtime = yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime;
  const secrets = yield* ServerSecretStore.ServerSecretStore;
  const httpClient = yield* HttpClient.HttpClient;
  // `order` ranks checks by start, so a slow older one cannot replace a newer result.
  const latestRef = yield* Ref.make(
    Option.none<{ readonly order: number; readonly result: EnvironmentConnectReachability }>(),
  );
  const checkCounter = yield* Ref.make(0);
  // Checks update this as they finish, even when a newer check already did. A dead streak
  // or a restart backoff that began before a healthy check no longer counts.
  const lastHealthyOrder = yield* Ref.make(0);
  const deadConnectorRef = yield* SynchronizedRef.make<{
    readonly streak: {
      readonly pid: number;
      readonly since: number;
      readonly order: number;
    } | null;
    readonly restartAfterMillis: number;
    readonly restartOrder: number;
  }>({
    streak: null,
    restartAfterMillis: Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
    restartOrder: 0,
  });
  const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
  const scope = yield* Scope.Scope;
  const readExitNode = readTailscaleExitNode.pipe(
    Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
    Effect.orElseSucceed(() => null),
  );

  const probeEndpoint = Effect.fn("ConnectReachability.probeEndpoint")(
    function* (hostname: string) {
      const response = yield* httpClient
        .get(`https://${hostname}/.well-known/t3/environment`, { acceptJson: true })
        .pipe(Effect.timeoutOption(ENDPOINT_REQUEST_TIMEOUT));
      if (Option.isNone(response)) return { kind: "timeout" } satisfies EndpointProbe;
      const { status } = response.value;
      if (status !== 530) return { kind: "status", status } satisfies EndpointProbe;
      // Cloudflare's error page is HTML unless the client accepts JSON, which carries
      // `"error_code":1033`. Clients that send no User-Agent get `error code: 1033`.
      const body = yield* response.value.text.pipe(
        Effect.timeoutOption(ERROR_BODY_READ_TIMEOUT),
        Effect.orElseSucceed(() => Option.none<string>()),
      );
      const code = /error[_ ]code"?:\s*(\d{4})/i.exec(Option.getOrElse(body, () => ""))?.[1];
      return (
        code === undefined
          ? { kind: "status", status }
          : { kind: "status", status, cloudflareError: code }
      ) satisfies EndpointProbe;
    },
    Effect.scoped,
    Effect.catch((error) =>
      Effect.succeed<EndpointProbe>({
        kind: hasErrorCode(error, DNS_ERROR_CODES) ? "dns_failed" : "network_error",
      }),
    ),
  );

  const gatherFacts = Effect.gen(function* () {
    const managedTunnelConfigured = Option.isSome(
      yield* secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG).pipe(Effect.orElseSucceed(Option.none)),
    );
    const connector = managedTunnelConfigured ? yield* runtime.inspectConnector : null;
    const endpoint =
      connector?.hostname && connector.readyConnections !== 0
        ? yield* probeEndpoint(connector.hostname)
        : null;
    const facts: ReachabilityFacts = {
      managedTunnelConfigured,
      connector,
      endpoint,
      edgeReachable: null,
      exitNode: null,
    };
    if (connector === null || (endpoint?.kind === "status" && endpoint.status === 200)) {
      return facts;
    }
    const [edgeResults, exitNode] = yield* Effect.all(
      [
        Effect.forEach(TUNNEL_EDGE_HOSTS, (host) => canOpenTcp(host, TUNNEL_EDGE_PORT), {
          concurrency: "unbounded",
        }),
        readExitNode,
      ],
      { concurrency: 2 },
    );
    return { ...facts, edgeReachable: edgeResults.some(Boolean), exitNode };
  });

  const isLatest = (order: number) =>
    Ref.get(latestRef).pipe(
      Effect.map((latest) => Option.isNone(latest) || latest.value.order <= order),
    );

  // Runs inside the check span; the runtime restart has its own. Returns the streak a dead
  // connector just started and when to confirm it.
  const restartDeadConnector = Effect.fnUntraced(function* (
    facts: ReachabilityFacts,
    result: EnvironmentConnectReachability,
    order: number,
  ) {
    const now = yield* Clock.currentTimeMillis;
    const { connector } = facts;
    const dead =
      result.status === "unreachable" &&
      result.cause === "no_connector" &&
      connector !== null &&
      (connector.readyConnections ?? 0) > 0
        ? connector
        : null;
    return yield* SynchronizedRef.modifyEffect(deadConnectorRef, (current) =>
      Effect.gen(function* () {
        const healthy = yield* Ref.get(lastHealthyOrder);
        const state = {
          streak: current.streak && current.streak.order > healthy ? current.streak : null,
          restartAfterMillis:
            current.restartOrder > healthy
              ? current.restartAfterMillis
              : Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER),
          restartOrder: current.restartOrder,
        };
        if (!(yield* isLatest(order))) return [null, state] as const;
        if (dead === null) return [null, { ...state, streak: null }] as const;
        if (state.streak?.pid !== dead.pid) {
          const started = { ...state, streak: { pid: dead.pid, since: now, order } };
          return [{ order, after: Duration.millis(state.restartAfterMillis) }, started] as const;
        }
        if (now - state.streak.since < state.restartAfterMillis) return [null, state] as const;
        // Checks can invalidate the streak while this request waits for the runtime's lock.
        const streakOrder = state.streak.order;
        const stillDead = Effect.gen(function* () {
          if (!(yield* isLatest(order))) return false;
          return (yield* Ref.get(lastHealthyOrder)) < streakOrder;
        });
        if (!(yield* runtime.restartConnector(dead.pid, stillDead))) {
          return [null, state] as const;
        }
        const restarted = {
          streak: null,
          restartAfterMillis: Math.min(
            state.restartAfterMillis * 2,
            Duration.toMillis(DEAD_CONNECTOR_RESTART_AFTER_MAX),
          ),
          restartOrder: order,
        };
        return [null, restarted] as const;
      }),
    );
  });

  const check: Effect.Effect<EnvironmentConnectReachability> = Effect.gen(function* () {
    const order = yield* Ref.updateAndGet(checkCounter, (count) => count + 1);
    const facts = yield* gatherFacts;
    const result = classifyReachability(facts, DateTime.formatIso(yield* DateTime.now));
    if (result.status === "reachable") {
      yield* Ref.update(lastHealthyOrder, (healthy) => Math.max(healthy, order));
    }
    const previous = yield* Ref.modify(latestRef, (current) =>
      Option.isSome(current) && current.value.order > order
        ? [null, current]
        : [current, Option.some({ order, result })],
    );
    if (previous === null) return result;
    const started = yield* restartDeadConnector(facts, result, order);
    // Nothing else would look again until the next periodic check.
    if (started !== null) {
      const confirm = SynchronizedRef.get(deadConnectorRef).pipe(
        Effect.flatMap(({ streak }) =>
          streak?.order === started.order ? Effect.suspend(() => check) : Effect.void,
        ),
      );
      yield* Effect.sleep(started.after).pipe(Effect.andThen(confirm), Effect.forkIn(scope));
    }
    const changed = Option.match(previous, {
      onNone: () => true,
      onSome: ({ result: last }) =>
        last.status !== result.status ||
        (last.status === "unreachable" &&
          result.status === "unreachable" &&
          last.cause !== result.cause),
    });
    if (!changed || (result.status === "unreachable" && result.cause === "not_linked")) {
      return result;
    }
    if (result.status === "unreachable") {
      // A connector that has not connected yet is usually still starting; the
      // runtime restarts it, and an unreachable edge is reported as its own cause.
      const log = result.cause === "connector_not_ready" ? Effect.logWarning : Effect.logError;
      yield* log(`\n${formatReachabilityFailure(result)}`).pipe(
        Effect.annotateLogs({ cause: result.cause }),
      );
    } else {
      yield* Effect.logInfo(
        `T3 Connect is reachable from other devices. ${formatConnectReachable(result)}`,
      );
    }
    return result;
  }).pipe(Effect.withSpan("ConnectReachability.check"));

  yield* runtime.readinessChanged.pipe(
    Stream.debounce(READINESS_CHANGE_DEBOUNCE),
    Stream.runForEach(() => check),
    Effect.forkScoped,
  );
  yield* Effect.sleep(RECHECK_INTERVAL).pipe(
    Effect.andThen(check),
    Effect.forever,
    Effect.forkScoped,
  );

  return ConnectReachability.of({
    check,
    latest: Ref.get(latestRef).pipe(Effect.map(Option.map(({ result }) => result))),
  });
});

export const layer = Layer.effect(ConnectReachability, make);