import * as Brand from "effect/Brand";
import * as Cause from "effect/Cause";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Option from "effect/Option";
import * as PlatformError from "effect/PlatformError";
import * as Ref from "effect/Ref";
import * as SubscriptionRef from "effect/SubscriptionRef";
import * as Schema from "effect/Schema";
import * as Semaphore from "effect/Semaphore";
import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
import { HttpClient } from "effect/http";
import { ChildProcess, ChildProcessSpawner } from "effect/process";
import {
SERVER_EXIT_CODE_MANAGED_POLICY,
SERVER_EXIT_CODE_STATE_DIR_OWNED,
DesktopBackendBootstrap,
type DesktopBackendBootstrap as DesktopBackendBootstrapValue,
PRIMARY_LOCAL_ENVIRONMENT_ID,
DesktopTelemetryControlMessage,
type DesktopTelemetryControlMessage as DesktopTelemetryControlMessageValue,
} from "@t3tools/contracts";
import { waitForHttpReady as waitForHttpReadyShared } from "@t3tools/shared/httpReadiness";
import * as DesktopObservability from "../app/DesktopObservability.ts";
import * as DesktopTelemetryPublisher from "../telemetry/DesktopTelemetryPublisher.ts";
import * as DesktopBrowserHost from "../preview/DesktopBrowserHost.ts";
import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts";
const INITIAL_RESTART_DELAY = Duration.millis(500);
const MAX_RESTART_DELAY = Duration.seconds(10);
const MAX_PREFLIGHT_FAILURE_ATTEMPTS = 5;
// Large state databases can take over a minute to become ready.
const DEFAULT_BACKEND_READINESS_TIMEOUT = Duration.minutes(3);
const DEFAULT_BACKEND_READINESS_INTERVAL = Duration.millis(100);
const DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT = Duration.seconds(1);
// Match the service launcher's grace period so backends can stop provider sessions.
const DEFAULT_BACKEND_TERMINATE_GRACE = Duration.seconds(5);
// Allow SIGKILL and the spawner exit wait after the SIGTERM grace so quit cannot
// orphan a backend whose shutdown hangs.
export const BACKEND_QUIT_STOP_TIMEOUT = Duration.sum(
DEFAULT_BACKEND_TERMINATE_GRACE,
Duration.seconds(2),
);
const DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT = Duration.seconds(5);
const BACKEND_READINESS_PATH = "/.well-known/t3/environment";
// Three restarts without readiness is about 2 s of backoff: past a blip, early enough to help.
const STARTUP_FAILURE_RESTART_ATTEMPTS = 3;
const STARTUP_FAILURE_OUTPUT_CHARS = 4_000;
const { logWarning: logBackendProcessWarning } =
DesktopObservability.makeComponentLogger("desktop-backend-process");
type BackendProcessLayerServices = ChildProcessSpawner.ChildProcessSpawner | HttpClient.HttpClient;
type BackendProcessRunRequirements = BackendProcessLayerServices | Scope.Scope;
export type BackendProcessOutputStream = "stdout" | "stderr";
export interface BackendProcessContext {
readonly executablePath: string;
readonly entryPath: string;
readonly cwd: string;
readonly httpBaseUrl: URL;
}
export type DesktopBackendBootstrapDelivery = "fd3" | "stdin";
export interface DesktopBackendStartConfig extends BackendProcessContext {
readonly args: ReadonlyArray<string>;
readonly env: Record<string, string | undefined>;
/** False passes env verbatim, preventing the Windows T3CODE_HOME from leaking into WSL. */
readonly extendEnv: boolean;
readonly bootstrap: DesktopBackendBootstrapValue;
readonly bootstrapDelivery: DesktopBackendBootstrapDelivery;
readonly httpBaseUrl: URL;
readonly captureOutput: boolean;
readonly preflightFailure: Option.Option<PreflightFailure>;
readonly runningDistro?: string;
readonly wslRuntimeId?: string;
}
export interface PreflightFailure {
readonly reason: string;
readonly fatal: boolean;
readonly retryLimit?: number;
}
interface BackendProcessExit {
readonly code: Option.Option<number>;
readonly reason: string;
}
const backendProcessContextSchema = {
executablePath: Schema.String,
entryPath: Schema.String,
cwd: Schema.String,
httpBaseUrl: Schema.URL,
};
export class BackendReadinessTimeoutError extends Schema.TaggedError<BackendReadinessTimeoutError>()(
"BackendReadinessTimeoutError",
{
...backendProcessContextSchema,
readinessUrl: Schema.URL,
timeoutMs: Schema.Number,
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Timed out after ${this.timeoutMs}ms waiting for desktop backend readiness at ${this.readinessUrl.href}.`;
}
}
export class BackendProcessBootstrapEncodeError extends Schema.TaggedError<BackendProcessBootstrapEncodeError>()(
"BackendProcessBootstrapEncodeError",
{
...backendProcessContextSchema,
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Failed to encode the desktop backend bootstrap payload for ${this.entryPath}.`;
}
}
export class BackendProcessSpawnError extends Schema.TaggedError<BackendProcessSpawnError>()(
"BackendProcessSpawnError",
{
...backendProcessContextSchema,
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Failed to spawn desktop backend entry ${this.entryPath} with ${this.executablePath}.`;
}
}
export class BackendProcessOutputReadError extends Schema.TaggedError<BackendProcessOutputReadError>()(
"BackendProcessOutputReadError",
{
...backendProcessContextSchema,
pid: Schema.Number,
streamName: Schema.Literals(["stdout", "stderr"]),
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Failed to read ${this.streamName} from desktop backend process ${this.pid}.`;
}
}
export class BackendProcessOutputHandlingError extends Schema.TaggedError<BackendProcessOutputHandlingError>()(
"BackendProcessOutputHandlingError",
{
...backendProcessContextSchema,
pid: Schema.Number,
streamName: Schema.Literals(["stdout", "stderr"]),
chunkByteLength: Schema.Number,
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Failed to handle ${this.chunkByteLength} bytes from ${this.streamName} of desktop backend process ${this.pid}.`;
}
}
export type BackendProcessOutputError =
| BackendProcessOutputReadError
| BackendProcessOutputHandlingError;
export class BackendProcessExitStatusError extends Schema.TaggedError<BackendProcessExitStatusError>()(
"BackendProcessExitStatusError",
{
...backendProcessContextSchema,
pid: Schema.Number,
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Failed to read the exit status of desktop backend process ${this.pid}.`;
}
}
export const BackendProcessError = Schema.Union([
BackendProcessBootstrapEncodeError,
BackendProcessSpawnError,
BackendProcessExitStatusError,
]);
export type BackendProcessError = typeof BackendProcessError.Type;
interface RunBackendProcessOptions extends DesktopBackendStartConfig {
readonly desktopTelemetryStream: Stream.Stream<Uint8Array>;
readonly desktopBrowserStream?: Stream.Stream<Uint8Array>;
readonly onDesktopBrowserCommand?: (line: string) => Effect.Effect<void>;
readonly onDesktopTelemetryControl?: (
message: DesktopTelemetryControlMessageValue,
) => Effect.Effect<void>;
readonly readinessTimeout?: Duration.Duration;
readonly outputDrainTimeout?: Duration.Duration;
readonly onStarted?: (pid: number) => Effect.Effect<void>;
readonly onExitObserved?: () => Effect.Effect<void>;
readonly onReady?: () => Effect.Effect<void>;
readonly onReadinessFailure?: (error: BackendReadinessTimeoutError) => Effect.Effect<void>;
readonly onOutput?: (
streamName: BackendProcessOutputStream,
chunk: Uint8Array,
) => Effect.Effect<void, Error>;
readonly onOutputFailure?: (error: BackendProcessOutputError) => Effect.Effect<void>;
}
export interface DesktopBackendSnapshot {
readonly desiredRunning: boolean;
readonly ready: boolean;
readonly activePid: Option.Option<number>;
readonly restartAttempt: number;
readonly restartScheduled: boolean;
}
export type BackendInstanceId = string & Brand.Brand<"BackendInstanceId">;
export const BackendInstanceId = Brand.nominal<BackendInstanceId>();
export const PRIMARY_INSTANCE_ID: BackendInstanceId = BackendInstanceId(
PRIMARY_LOCAL_ENVIRONMENT_ID,
);
export interface BackendStartupFailure {
readonly reason: string;
/** The server refused to start under this machine's managed policy. */
readonly managedPolicy: boolean;
/** Tail of the last run's stdout and stderr. */
readonly output: string;
}
export interface DesktopBackendInstance {
readonly id: BackendInstanceId;
readonly label: Effect.Effect<string>;
readonly start: Effect.Effect<void>;
readonly stop: (options?: { readonly timeout?: Duration.Duration }) => Effect.Effect<void>;
readonly currentConfig: Effect.Effect<Option.Option<DesktopBackendStartConfig>>;
readonly snapshot: Effect.Effect<DesktopBackendSnapshot>;
readonly waitForReady: (timeout: Duration.Duration) => Effect.Effect<boolean>;
}
export interface BackendInstanceSpec {
readonly id: BackendInstanceId;
readonly label: Effect.Effect<string>;
readonly configResolve: Effect.Effect<DesktopBackendStartConfig, PlatformError.PlatformError>;
/** Receives the ready run's resolved URL; reading current config could race a subsequent restart. */
readonly onReady?: (httpBaseUrl: URL) => Effect.Effect<void>;
readonly onShutdown?: () => Effect.Effect<void>;
readonly onPreflightFailed?: (failure: PreflightFailure) => Effect.Effect<boolean>;
readonly onStateDirOwned?: () => Effect.Effect<void>;
/** Fires on each repeated crash or readiness timeout until the backend becomes ready. */
readonly onStartupFailure?: (failure: BackendStartupFailure) => Effect.Effect<void>;
}
interface ActiveBackendRun {
readonly id: number;
readonly scope: Scope.Closeable;
readonly fiber: Option.Option<Fiber.Fiber<void, never>>;
readonly pid: Option.Option<number>;
readonly exitObserved: boolean;
readonly stopRequested: boolean;
}
interface BackendManagerState {
readonly desiredRunning: boolean;
readonly ready: boolean;
readonly config: Option.Option<DesktopBackendStartConfig>;
readonly active: Option.Option<ActiveBackendRun>;
readonly restartAttempt: number;
readonly preflightFailureAttempt: number;
readonly restartFiber: Option.Option<Fiber.Fiber<void, never>>;
readonly nextRunId: number;
}
const initialState: BackendManagerState = {
desiredRunning: false,
ready: false,
config: Option.none(),
active: Option.none(),
restartAttempt: 0,
preflightFailureAttempt: 0,
restartFiber: Option.none(),
nextRunId: 1,
};
const activePid = (active: Option.Option<ActiveBackendRun>): Option.Option<number> =>
Option.flatMap(active, (run) => run.pid);
const withActiveRun =
(runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) =>
(state: BackendManagerState): BackendManagerState => ({
...state,
active: Option.map(state.active, (run) => (run.id === runId ? f(run) : run)),
});
const calculateRestartDelay = (attempt: number): Duration.Duration =>
Duration.min(Duration.times(INITIAL_RESTART_DELAY, 2 ** attempt), MAX_RESTART_DELAY);
const closeRun = (
run: ActiveBackendRun,
parentScope: Scope.Scope,
options?: { readonly timeout?: Duration.Duration },
): Effect.Effect<boolean> => {
const waitForFiber = Option.match(run.fiber, {
onNone: () => Effect.void,
onSome: (fiber) => Fiber.await(fiber).pipe(Effect.asVoid),
});
const close = Scope.close(run.scope, Exit.void).pipe(
Effect.withSpan("desktop.backendInstance.closeScope"),
Effect.andThen(waitForFiber.pipe(Effect.withSpan("desktop.backendInstance.awaitRun"))),
);
const timeout = options?.timeout;
if (!timeout) {
return close.pipe(Effect.as(true));
}
return Effect.forkIn(close, parentScope).pipe(
Effect.flatMap((closeFiber) =>
Fiber.await(closeFiber).pipe(Effect.timeoutOption(timeout), Effect.map(Option.isSome)),
),
);
};
const waitForHttpReady = (
options: BackendProcessContext & { readonly timeout: Duration.Duration },
): Effect.Effect<void, BackendReadinessTimeoutError, HttpClient.HttpClient> => {
const readinessUrl = new URL(BACKEND_READINESS_PATH, options.httpBaseUrl);
return waitForHttpReadyShared({
baseUrl: options.httpBaseUrl.href,
path: BACKEND_READINESS_PATH,
timeoutMs: Duration.toMillis(options.timeout),
intervalMs: Duration.toMillis(DEFAULT_BACKEND_READINESS_INTERVAL),
probeTimeoutMs: Duration.toMillis(DEFAULT_BACKEND_READINESS_REQUEST_TIMEOUT),
makeError: ({ cause }) =>
new BackendReadinessTimeoutError({
executablePath: options.executablePath,
entryPath: options.entryPath,
cwd: options.cwd,
httpBaseUrl: options.httpBaseUrl,
readinessUrl,
timeoutMs: Duration.toMillis(options.timeout),
cause,
}),
});
};
function drainBackendOutput(
context: BackendProcessContext & { readonly pid: number },
streamName: BackendProcessOutputStream,
stream: Stream.Stream<Uint8Array, PlatformError.PlatformError>,
onOutput: (
streamName: BackendProcessOutputStream,
chunk: Uint8Array,
) => Effect.Effect<void, Error>,
onOutputFailure: (error: BackendProcessOutputError) => Effect.Effect<void>,
): Effect.Effect<void> {
return stream.pipe(
Stream.mapError(
(cause) =>
new BackendProcessOutputReadError({
...context,
streamName,
cause,
}),
),
Stream.runForEach((chunk) =>
onOutput(streamName, chunk).pipe(
Effect.mapError(
(cause) =>
new BackendProcessOutputHandlingError({
...context,
streamName,
chunkByteLength: chunk.byteLength,
cause,
}),
),
Effect.catchTags({ BackendProcessOutputHandlingError: onOutputFailure }),
),
),
Effect.catchTags({
BackendProcessOutputReadError: onOutputFailure,
}),
);
}
const encodeBootstrapJson = Schema.encodeEffect(Schema.fromJsonString(DesktopBackendBootstrap));
const decodeDesktopTelemetryControlLine = Schema.decodeUnknownEffect(
Schema.fromJsonString(DesktopTelemetryControlMessage),
);
export const runBackendProcess = Effect.fn("runBackendProcess")(function* (
options: RunBackendProcessOptions,
): Effect.fn.Return<BackendProcessExit, BackendProcessError, BackendProcessRunRequirements> {
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
const bootstrapJson = yield* encodeBootstrapJson(options.bootstrap).pipe(
Effect.mapError(
(cause) =>
new BackendProcessBootstrapEncodeError({
executablePath: options.executablePath,
entryPath: options.entryPath,
cwd: options.cwd,
httpBaseUrl: options.httpBaseUrl,
cause,
}),
),
);
const onOutput = options.onOutput ?? (() => Effect.void);
const bootstrapStream = Stream.encodeText(Stream.make(`${bootstrapJson}\n`));
const additionalFds: Record<`fd${number}`, ChildProcess.AdditionalFdConfig> = {};
if (options.bootstrapDelivery === "fd3") {
additionalFds.fd3 = {
type: "input",
stream: bootstrapStream,
};
if (options.bootstrap.desktopTelemetryFd !== undefined) {
additionalFds[`fd${options.bootstrap.desktopTelemetryFd}`] = {
type: "input",
stream: options.desktopTelemetryStream,
};
}
if (options.bootstrap.desktopTelemetryControlFd !== undefined) {
additionalFds[`fd${options.bootstrap.desktopTelemetryControlFd}`] = {
type: "output",
};
}
if (
options.bootstrap.desktopBrowserFd !== undefined &&
options.desktopBrowserStream !== undefined
) {
additionalFds[`fd${options.bootstrap.desktopBrowserFd}`] = {
type: "input",
stream: options.desktopBrowserStream,
};
}
if (options.bootstrap.desktopBrowserControlFd !== undefined) {
additionalFds[`fd${options.bootstrap.desktopBrowserControlFd}`] = { type: "output" };
}
}
const command = ChildProcess.make(options.executablePath, options.args, {
cwd: options.cwd,
env: options.env,
extendEnv: options.extendEnv,
stdin: options.bootstrapDelivery === "stdin" ? bootstrapStream : "ignore",
stdout: options.captureOutput ? "pipe" : "inherit",
stderr: options.captureOutput ? "pipe" : "inherit",
killSignal: "SIGTERM",
forceKillAfter: DEFAULT_BACKEND_TERMINATE_GRACE,
// wsl.exe drops extra file descriptors; deliver its bootstrap via stdin.
...(options.bootstrapDelivery === "fd3" ? { additionalFds } : {}),
});
const handle = yield* spawner.spawn(command).pipe(
Effect.mapError(
(cause) =>
new BackendProcessSpawnError({
executablePath: options.executablePath,
entryPath: options.entryPath,
cwd: options.cwd,
httpBaseUrl: options.httpBaseUrl,
cause,
}),
),
);
const outputFibers: Array<Fiber.Fiber<void, never>> = [];
yield* options.onStarted?.(handle.pid) ?? Effect.void;
if (
options.bootstrap.desktopTelemetryControlFd !== undefined &&
options.onDesktopTelemetryControl !== undefined
) {
const controlFd = options.bootstrap.desktopTelemetryControlFd;
const handleControl = options.onDesktopTelemetryControl;
yield* handle.getOutputFd(controlFd).pipe(
Stream.decodeText(),
Stream.splitLines,
Stream.filter((line) => line.trim().length > 0),
Stream.runForEach((line) =>
decodeDesktopTelemetryControlLine(line).pipe(
Effect.flatMap(handleControl),
Effect.catchCause((cause) =>
logBackendProcessWarning("ignored invalid desktop telemetry control message", {
fd: controlFd,
cause: Cause.pretty(cause),
}),
),
),
),
Effect.catchCause((cause) =>
logBackendProcessWarning("desktop telemetry control stream stopped", {
fd: controlFd,
cause: Cause.pretty(cause),
}),
),
Effect.ensuring(
handleControl({
version: 1,
type: "setDiagnosticsDemand",
enabled: false,
}),
),
Effect.forkScoped,
);
}
if (
options.bootstrap.desktopBrowserControlFd !== undefined &&
options.onDesktopBrowserCommand !== undefined
) {
const browserFd = options.bootstrap.desktopBrowserControlFd;
const handleCommand = options.onDesktopBrowserCommand;
yield* handle.getOutputFd(browserFd).pipe(
Stream.decodeText(),
Stream.splitLines,
Stream.filter((line) => line.length > 0),
Stream.runForEach(handleCommand),
Effect.catchCause((cause) =>
logBackendProcessWarning("desktop browser command stream stopped", {
fd: browserFd,
cause: Cause.pretty(cause),
}),
),
Effect.forkScoped,
);
}
if (options.captureOutput) {
const outputContext = {
executablePath: options.executablePath,
entryPath: options.entryPath,
cwd: options.cwd,
httpBaseUrl: options.httpBaseUrl,
pid: Number(handle.pid),
};
const onOutputFailure = options.onOutputFailure ?? (() => Effect.void);
outputFibers.push(
yield* drainBackendOutput(
outputContext,
"stdout",
handle.stdout,
onOutput,
onOutputFailure,
).pipe(Effect.forkScoped),
yield* drainBackendOutput(
outputContext,
"stderr",
handle.stderr,
onOutput,
onOutputFailure,
).pipe(Effect.forkScoped),
);
}
// Cold WSL boots across /mnt/c can exceed one probe budget; retry while the backend is alive.
const probeReadiness = Effect.fn("desktop.backendProcess.probeReadiness")(() =>
waitForHttpReady({
executablePath: options.executablePath,
entryPath: options.entryPath,
cwd: options.cwd,
httpBaseUrl: options.httpBaseUrl,
timeout: options.readinessTimeout ?? DEFAULT_BACKEND_READINESS_TIMEOUT,
}).pipe(
Effect.flatMap(() => options.onReady?.() ?? Effect.void),
Effect.as(true),
Effect.catchTags({
BackendReadinessTimeoutError: (error) =>
(options.onReadinessFailure?.(error) ?? Effect.void).pipe(Effect.as(false)),
}),
),
);
yield* probeReadiness().pipe(Effect.repeat({ while: (ready) => !ready }), Effect.forkScoped);
const exit = yield* handle.exitCode.pipe(
Effect.withSpan("desktop.backendProcess.awaitExit"),
Effect.map((exitCode) => ({
code: Option.some(exitCode),
reason: `code=${exitCode}`,
})),
Effect.catch((cause) => {
// Effect's Node spawner exposes signal exits only through this cause message.
const signal =
cause.cause instanceof Error
? /^Process interrupted due to receipt of signal: '(SIG[A-Z0-9]+)'$/.exec(
cause.cause.message,
)?.[1]
: undefined;
if (signal !== undefined) {
return Effect.succeed({ code: Option.none(), reason: `killed by ${signal}` });
}
return Effect.fail(
new BackendProcessExitStatusError({
executablePath: options.executablePath,
entryPath: options.entryPath,
cwd: options.cwd,
httpBaseUrl: options.httpBaseUrl,
pid: Number(handle.pid),
cause,
}),
);
}),
Effect.exit,
);
yield* options.onExitObserved?.() ?? Effect.void;
yield* Effect.forEach(outputFibers, Fiber.await, {
concurrency: "unbounded",
discard: true,
}).pipe(
Effect.timeout(options.outputDrainTimeout ?? DEFAULT_BACKEND_OUTPUT_DRAIN_TIMEOUT),
Effect.ignore,
Effect.withSpan("desktop.backendProcess.drainOutput"),
);
if (Exit.isFailure(exit)) {
return yield* Effect.failCause(exit.cause);
}
return exit.value;
});
export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
spec: BackendInstanceSpec,
): Effect.fn.Return<
DesktopBackendInstance,
never,
| FileSystem.FileSystem
| ChildProcessSpawner.ChildProcessSpawner
| HttpClient.HttpClient
| DesktopObservability.DesktopBackendOutputLogFactory
| DesktopTelemetryPublisher.DesktopTelemetryPublisher
| DesktopBrowserHost.DesktopBrowserHost
| DesktopWslEnvironment.DesktopWslEnvironment
| Scope.Scope
> {
const parentScope = yield* Scope.Scope;
const fileSystem = yield* FileSystem.FileSystem;
const backendOutputLogFactory = yield* DesktopObservability.DesktopBackendOutputLogFactory;
const backendOutputLog = yield* backendOutputLogFactory.forInstance(spec.id);
const desktopTelemetryPublisher = yield* DesktopTelemetryPublisher.DesktopTelemetryPublisher;
const desktopBrowserHost = yield* DesktopBrowserHost.DesktopBrowserHost;
const wslEnvironment = yield* DesktopWslEnvironment.DesktopWslEnvironment;
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
const httpClient = yield* HttpClient.HttpClient;
const state = yield* SubscriptionRef.make(initialState);
const mutex = yield* Semaphore.make(1);
const outputTail = yield* Ref.make("");
// One streaming decoder per stream keeps UTF-8 characters split across chunks intact.
const makeOutputDecoders = () => ({ stdout: new TextDecoder(), stderr: new TextDecoder() });
let outputDecoders = makeOutputDecoders();
const reportStartupFailure = (reason: string, managedPolicy = false) =>
Ref.get(outputTail).pipe(
Effect.flatMap(
(output) => spec.onStartupFailure?.({ reason, managedPolicy, output }) ?? Effect.void,
),
);
const { logWarning: logInstanceWarning, logError: logInstanceError } =
DesktopObservability.makeComponentLogger(`desktop-backend-instance:${spec.id}`);
const updateActiveRun = (runId: number, f: (run: ActiveBackendRun) => ActiveBackendRun) =>
SubscriptionRef.update(state, withActiveRun(runId, f));
const snapshot = SubscriptionRef.get(state).pipe(
Effect.map((current): DesktopBackendSnapshot => ({
desiredRunning: current.desiredRunning,
ready: current.ready,
activePid: activePid(current.active),
restartAttempt: current.restartAttempt,
restartScheduled: Option.isSome(current.restartFiber),
})),
);
const currentConfig = SubscriptionRef.get(state).pipe(Effect.map((current) => current.config));
const cancelRestart = Effect.gen(function* () {
const restartFiber = yield* SubscriptionRef.modify(state, (current) => [
current.restartFiber,
{
...current,
restartFiber: Option.none(),
},
]);
yield* Option.match(restartFiber, {
onNone: () => Effect.void,
onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid),
});
});
const start: Effect.Effect<void> = Effect.suspend(() =>
mutex.withPermits(1)(
Effect.gen(function* () {
const current = yield* SubscriptionRef.get(state);
if (Option.isSome(current.active)) {
if (!current.desiredRunning) {
yield* SubscriptionRef.update(state, (latest) => ({
...latest,
desiredRunning: true,
}));
}
return;
}
if (current.ready) {
yield* spec.onShutdown?.() ?? Effect.void;
yield* SubscriptionRef.update(state, (latest) =>
latest.ready ? { ...latest, ready: false } : latest,
);
}
const config = yield* spec.configResolve.pipe(
Effect.tapError((error) =>
logInstanceError("failed to generate desktop backend configuration", {
cause: error.message,
}),
),
Effect.option,
);
if (Option.isNone(config)) {
if (current.desiredRunning) {
yield* scheduleRestart("failed to generate desktop backend configuration");
}
return;
}
const entryExists = yield* fileSystem
.exists(config.value.entryPath)
.pipe(Effect.orElseSucceed(() => false));
const resetFatalPreflightCounter =
!current.desiredRunning && current.preflightFailureAttempt > 0;
yield* cancelRestart;
yield* SubscriptionRef.update(state, (latest) => ({
...latest,
desiredRunning: true,
ready: false,
config: Option.some(config.value),
preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt,
}));
const preflightFailure = config.value.preflightFailure;
if (Option.isSome(preflightFailure)) {
const { reason, fatal, retryLimit } = preflightFailure.value;
if (!fatal && retryLimit === undefined) {
yield* SubscriptionRef.update(state, (latest) =>
latest.preflightFailureAttempt === 0
? latest
: { ...latest, preflightFailureAttempt: 0 },
);
yield* scheduleRestart(reason);
return;
}
const attemptLimit = retryLimit ?? MAX_PREFLIGHT_FAILURE_ATTEMPTS;
const attempt = yield* SubscriptionRef.modify(state, (latest) => {
const next = latest.preflightFailureAttempt + 1;
return [next, { ...latest, preflightFailureAttempt: next }] as const;
});
if (attempt > attemptLimit) {
yield* logInstanceError("backend preflight still failing after fallback; stopping", {
reason,
attempt,
});
yield* SubscriptionRef.update(state, (latest) => ({
...latest,
desiredRunning: false,
ready: false,
}));
return;
}
if (attempt === attemptLimit) {
yield* logInstanceError(
"backend preflight failed repeatedly; surfacing and falling back",
{ reason, attempt },
);
const shouldRestart = yield* (
spec.onPreflightFailed?.(preflightFailure.value) ?? Effect.succeed(false)
);
if (shouldRestart) {
yield* scheduleRestart(reason);
} else {
yield* SubscriptionRef.update(state, (latest) => ({
...latest,
desiredRunning: false,
ready: false,
}));
}
return;
}
yield* scheduleRestart(reason);
return;
}
yield* SubscriptionRef.update(state, (latest) =>
latest.preflightFailureAttempt === 0 ? latest : { ...latest, preflightFailureAttempt: 0 },
);
if (!entryExists) {
yield* scheduleRestart(`missing server entry at ${config.value.entryPath}`);
return;
}
const runScope = yield* Scope.make("sequential");
const runId = yield* SubscriptionRef.modify(state, (latest) => [
latest.nextRunId,
{
...latest,
active: Option.some({
id: latest.nextRunId,
scope: runScope,
fiber: Option.none(),
pid: Option.none(),
exitObserved: false,
stopRequested: false,
} satisfies ActiveBackendRun),
nextRunId: latest.nextRunId + 1,
},
]);
const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* (
reason: string,
exitCode?: number,
) {
const stateDirOwned = exitCode === SERVER_EXIT_CODE_STATE_DIR_OWNED;
yield* mutex.withPermits(1)(
Effect.gen(function* () {
const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } =
yield* SubscriptionRef.modify(
state,
(
latest,
): readonly [
{
readonly isCurrentRun: boolean;
readonly nextState: BackendManagerState;
readonly pid: Option.Option<number>;
readonly exitObserved: boolean;
readonly stopRequested: boolean;
readonly wasReady: boolean;
},
BackendManagerState,
] => {
const currentRun = Option.getOrUndefined(latest.active);
if (currentRun?.id !== runId) {
return [
{
isCurrentRun: false,
nextState: latest,
pid: Option.none<number>(),
exitObserved: false,
stopRequested: false,
wasReady: false,
},
latest,
] as const;
}
const next = {
...latest,
active: Option.none<ActiveBackendRun>(),
ready: false,
};
return [
{
isCurrentRun: true,
nextState: next,
pid: currentRun.pid,
exitObserved: currentRun.exitObserved,
stopRequested: currentRun.stopRequested,
wasReady: latest.ready,
},
next,
] as const;
},
);
if (isCurrentRun) {
yield* desktopTelemetryPublisher.removeControlSource(spec.id);
if (Option.isSome(pid)) {
if (exitObserved && !stopRequested) {
yield* backendOutputLog.persistFailure({
details: `pid=${pid.value} ${reason}`,
});
} else {
yield* backendOutputLog.discardSession;
}
}
if (wasReady) {
yield* spec.onShutdown?.() ?? Effect.void;
}
}
if (isCurrentRun && nextState.desiredRunning) {
if (stateDirOwned && !stopRequested) {
yield* SubscriptionRef.update(state, (latest) => ({
...latest,
desiredRunning: false,
}));
yield* (spec.onStateDirOwned?.() ?? Effect.void).pipe(Effect.ignore);
return;
}
yield* scheduleRestart(reason, exitCode);
}
}),
);
});
const program = runBackendProcess({
...config.value,
desktopTelemetryStream: desktopTelemetryPublisher.encoded,
desktopBrowserStream: desktopBrowserHost.events,
onDesktopBrowserCommand: desktopBrowserHost.handleCommandLine,
onDesktopTelemetryControl: (message) =>
desktopTelemetryPublisher.handleControlForSource(spec.id, message),
onStarted: Effect.fn("desktop.backendInstance.onStarted")(function* (pid) {
yield* updateActiveRun(runId, (run) => ({
...run,
pid: Option.some(pid),
}));
yield* Ref.set(outputTail, "");
outputDecoders = makeOutputDecoders();
yield* backendOutputLog.beginSession({
details: `pid=${pid} port=${config.value.bootstrap.port} cwd=${config.value.cwd}`,
});
}),
onExitObserved: () =>
updateActiveRun(runId, (run) => ({
...run,
exitObserved: true,
})),
onReady: Effect.fn("desktop.backendInstance.onReady")(function* () {
const isCurrentRun = yield* SubscriptionRef.modify(state, (latest) => {
const activeRun = Option.getOrUndefined(latest.active);
if (activeRun?.id !== runId) {
return [false, latest] as const;
}
return [
true,
{
...latest,
restartAttempt: 0,
ready: true,
},
] as const;
});
if (!isCurrentRun) {
return;
}
yield* spec.onReady?.(config.value.httpBaseUrl) ?? Effect.void;
if (
config.value.runningDistro !== undefined &&
config.value.wslRuntimeId !== undefined
) {
yield* wslEnvironment.pruneRuntimes(
config.value.runningDistro,
config.value.wslRuntimeId,
);
}
}),
onReadinessFailure: Effect.fn("desktop.backendInstance.onReadinessFailure")(
function* (error) {
yield* logInstanceWarning("backend readiness check failed during bootstrap", {
error: error.message,
});
yield* backendOutputLog.persistFailureSnapshot({
details: error.message,
});
yield* reportStartupFailure("The server started but did not become ready in time.");
},
),
onOutput: (streamName, chunk) =>
Ref.update(outputTail, (tail) =>
// A cut through a surrogate pair would make encodeURIComponent throw when rendering.
(tail + outputDecoders[streamName].decode(chunk, { stream: true }))
.slice(-STARTUP_FAILURE_OUTPUT_CHARS)
.toWellFormed(),
).pipe(Effect.andThen(backendOutputLog.writeOutputChunk(streamName, chunk))),
}).pipe(
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner),
Effect.provideService(HttpClient.HttpClient, httpClient),
Scope.provide(runScope),
Effect.matchEffect({
onFailure: (error) => finalizeRun(error.message),
onSuccess: (exit) => finalizeRun(exit.reason, Option.getOrUndefined(exit.code)),
}),
Effect.ensuring(Scope.close(runScope, Exit.void).pipe(Effect.ignore)),
);
const fiber = yield* Effect.forkIn(program, parentScope);
yield* updateActiveRun(runId, (run) => ({
...run,
fiber: Option.some(fiber),
}));
}),
),
).pipe(Effect.withSpan("desktop.backendInstance.start", { attributes: { id: spec.id } }));
const scheduleRestart = Effect.fn("desktop.backendInstance.scheduleRestart")(function* (
reason: string,
exitCode?: number,
) {
const scheduled = yield* SubscriptionRef.modify(state, (latest) => {
if (!latest.desiredRunning || Option.isSome(latest.restartFiber)) {
return [Option.none<{ delay: Duration.Duration; attempt: number }>(), latest] as const;
}
const attempt = latest.restartAttempt + 1;
return [
Option.some({ delay: calculateRestartDelay(latest.restartAttempt), attempt }),
{ ...latest, restartAttempt: attempt },
] as const;
});
yield* Option.match(scheduled, {
onNone: () => Effect.void,
onSome: Effect.fn("desktop.backendInstance.scheduleRestartFiber")(function* ({
delay,
attempt,
}) {
yield* logInstanceError("backend exited unexpectedly; restart scheduled", {
reason,
delayMs: Duration.toMillis(delay),
});
// Retrying cannot fix a policy refusal, so say so at once; retries still pick up a fix.
const managedPolicy = exitCode === SERVER_EXIT_CODE_MANAGED_POLICY;
if (managedPolicy || attempt >= STARTUP_FAILURE_RESTART_ATTEMPTS) {
yield* reportStartupFailure(`The server could not start (${reason}).`, managedPolicy);
}
const restartFiber = yield* Effect.forkIn(
Effect.sleep(delay).pipe(
Effect.andThen(
SubscriptionRef.modify(state, (latest) => {
const shouldRestart = latest.desiredRunning;
return [
shouldRestart,
{
...latest,
restartFiber: Option.none(),
},
] as const;
}),
),
Effect.flatMap((shouldRestart) => (shouldRestart ? start : Effect.void)),
Effect.catchCause((cause) =>
logInstanceError("desktop backend restart fiber failed", {
cause: Cause.pretty(cause),
}),
),
),
parentScope,
);
yield* SubscriptionRef.update(state, (latest) =>
Option.isNone(latest.restartFiber)
? {
...latest,
restartFiber: Option.some(restartFiber),
}
: latest,
);
}),
});
});
const stop = Effect.fn("desktop.backendInstance.stop")(function* (options?: {
readonly timeout?: Duration.Duration;
}) {
const { active, restartFiber, notifyShutdown } = yield* mutex.withPermits(1)(
Effect.gen(function* () {
const result = yield* SubscriptionRef.modify(state, (latest) => {
const active = Option.map(latest.active, (run) =>
run.exitObserved ? run : { ...run, stopRequested: true },
);
return [
{
active,
restartFiber: latest.restartFiber,
notifyShutdown: latest.ready,
},
{
...latest,
desiredRunning: false,
ready: false,
active,
restartFiber: Option.none<Fiber.Fiber<void, never>>(),
},
] as const;
});
return result;
}),
);
if (notifyShutdown) {
yield* (spec.onShutdown?.() ?? Effect.void).pipe(Effect.ignore);
}
yield* Option.match(restartFiber, {
onNone: () => Effect.void,
onSome: (fiber) => Fiber.interrupt(fiber).pipe(Effect.asVoid),
});
yield* Option.match(active, {
onNone: () => Effect.void,
// The enclosing operation owns the span for this inline handler.
onSome: Effect.fnUntraced(function* (run) {
const closed = yield* closeRun(run, parentScope, options);
if (!closed) {
return;
}
const cleanup = yield* mutex.withPermits(1)(
SubscriptionRef.modify(
state,
(
latest,
): readonly [
{
readonly needsCleanup: boolean;
readonly shouldStart: boolean;
},
BackendManagerState,
] => {
const current = Option.getOrUndefined(latest.active);
if (current?.id !== run.id) {
return [
{
needsCleanup: false,
shouldStart:
latest.desiredRunning &&
Option.isNone(latest.active) &&
Option.isNone(latest.restartFiber),
},
latest,
];
}
return [
{
needsCleanup: true,
shouldStart: latest.desiredRunning,
},
{
...latest,
active: Option.none<ActiveBackendRun>(),
},
];
},
),
);
if (cleanup.needsCleanup) {
yield* desktopTelemetryPublisher.removeControlSource(spec.id);
yield* backendOutputLog.discardSession;
}
if (cleanup.shouldStart) {
yield* start;
}
}),
});
});
const waitForReady = Effect.fn("DesktopBackendManager.waitForReady")(
(timeout: Duration.Duration) =>
SubscriptionRef.changes(state).pipe(
Stream.filter((current) => current.ready || !current.desiredRunning),
Stream.map((current) => current.ready),
Stream.runHead,
Effect.timeoutOption(timeout),
Effect.map(
(result) => Option.isSome(result) && Option.getOrElse(result.value, () => false),
),
),
);
yield* Effect.addFinalizer(() => stop());
return {
id: spec.id,
label: spec.label,
start,
stop,
currentConfig,
snapshot,
waitForReady,
} satisfies DesktopBackendInstance;
});