sailfork / t3code-github / apps/ server/ src/ process/ agentScope.ts

// @effect-diagnostics nodeBuiltinImport:off -- unit names and RAM size come from the host.
import * as NodeCrypto from "node:crypto";
import * as NodeOS from "node:os";

import {
  AgentScope,
  agentScopeCommand,
  type AgentScopeShape,
  type AgentSystemdScope,
} from "@t3tools/shared/AgentScope";
import * as HostProcess from "@t3tools/shared/HostProcess";
import { resolveCommandPath } from "@t3tools/shared/shell";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";

import * as ServerConfig from "../config.ts";
import * as ProcessRunner from "../processRunner.ts";

const AGENT_SLICE = "app-t3code-agents.slice";
// Owns every agent slice and competes with the server's unit or scope under app.slice.
const AGENTS_PARENT_SLICE = "app-t3code.slice";
// Low weights yield CPU and disk to the server under contention, where the user manager
// has those controllers: current systemd delegates cpu, rarely io. TasksMax keeps agents
// below logind's 33% per-user task limit, leaving forks for the server.
const AGENTS_PARENT_PROPERTIES = ["CPUWeight=10", "IOWeight=10", "TasksMax=25%"];

export function ownedAgentSlice(stateDir: string): string {
  const hash = NodeCrypto.createHash("sha256").update(stateDir).digest("hex").slice(0, 12);
  return `app-t3code-agents-${hash}.slice`;
}

const GiB = 1024 ** 3;

/**
 * Agents together leave about 6 GB free before they are throttled and 4 GB
 * before the kernel kills one, so the server, sshd, and tailscaled keep
 * working. Small machines still give agents at least half of RAM before
 * throttling and three quarters before a kill.
 */
function agentSliceMemoryLimits(totalBytes: number) {
  return {
    high: Math.max(totalBytes - 6 * GiB, Math.floor(totalBytes / 2)),
    max: Math.max(totalBytes - 4 * GiB, Math.floor(totalBytes * 0.75)),
  };
}

export interface ScopeState {
  readonly loadState: string;
  readonly activeState: string;
  readonly result: string;
  readonly oomKills: number | undefined;
  readonly populated: boolean | undefined;
}

/** Pipes can close before systemd records oom-kill; stopping, empty or OOM-counted scopes remain pending. */
export function classifyScope(state: ScopeState): "oom-killed" | "gone" | "running" | "stopping" {
  if (state.result === "oom-kill") return "oom-killed";
  if (
    state.loadState !== "loaded" ||
    state.activeState === "inactive" ||
    state.activeState === "failed"
  ) {
    return "gone";
  }
  if (
    state.activeState === "deactivating" ||
    (state.oomKills ?? 0) > 0 ||
    state.populated === false
  ) {
    return "stopping";
  }
  return "running";
}

function parseKeyValues(text: string): Map<string, string> {
  const values = new Map<string, string>();
  for (const line of text.split("\n")) {
    const index = line.indexOf("=");
    if (index > 0) values.set(line.slice(0, index), line.slice(index + 1).trim());
  }
  return values;
}

function parseCgroupCounter(text: string, key: string): number | undefined {
  const match = new RegExp(`^${key} (\\d+)

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	apps/server/src/process/agentScope.ts at main · sailfork/t3code-github · Sailfork
	
		
, "m").exec(text); return match === null ? undefined : Number(match[1]); } // systemd's default 90 s stop timeout would outlast our leftover-agent wait. // Delegation lets rootless containers (Podman, kind) manage their own cgroups. const SCOPE_OPTIONS = ["--property=TimeoutStopSec=10s", "--property=Delegate=yes"]; const STOPPING_POLL_INTERVAL = "100 millis"; const STOPPING_POLL_ATTEMPTS = 20; const CLEAR_FAILED_INTERVAL = "1 second"; const CLEAR_FAILED_ATTEMPTS = 120; const MAX_TRACKED_THREADS = 500; export const make = Effect.gen(function* () { const platform = yield* HostProcess.Platform; if (platform !== "linux") return AgentScope.defaultValue(); const runner = yield* ProcessRunner.ProcessRunner; const fileSystem = yield* FileSystem.FileSystem; const path = yield* Path.Path; const hostEnvironment = yield* HostProcess.Environment; const { stateDir } = yield* ServerConfig.ServerConfig; const ownedSlice = ownedAgentSlice(stateDir); const layerScope = yield* Effect.scope; const resolve = (command: string, env: NodeJS.ProcessEnv) => resolveCommandPath(command, { env }).pipe( Effect.provideService(FileSystem.FileSystem, fileSystem), Effect.provideService(Path.Path, path), Effect.provideService(HostProcess.Platform, platform), ); const run = (command: string, args: ReadonlyArray<string>) => runner.run({ command, args, timeout: "5 seconds" }).pipe(Effect.option); const readCgroupFile = (controlGroup: string, file: string) => fileSystem .readFileString(path.join("/sys/fs/cgroup", controlGroup, file)) .pipe(Effect.orElseSucceed(() => "")); // Old agents keep their scopes when the server is killed, and the restarted // server resumes their sessions. Stop them first, so two copies of one // session never work in the same worktree. const stopLeftoverAgents = Effect.fn("agentScope.stopLeftoverAgents")(function* ( systemctl: string, ) { const shown = yield* run(systemctl, [ "--user", "show", ownedSlice, "--property=ActiveState,ControlGroup", ]); if (shown._tag === "None") return; const values = parseKeyValues(shown.value.stdout); if (values.get("ActiveState") !== "active") return; const controlGroup = values.get("ControlGroup") ?? ""; const events = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : ""; if (parseCgroupCounter(events, "populated") === 0) return; yield* Effect.logWarning("Stopping agents left by a previous server", { slice: ownedSlice }); const stopped = yield* runner .run({ command: systemctl, args: ["--user", "stop", ownedSlice], timeout: "30 seconds" }) .pipe(Effect.option); if (stopped._tag === "None" || stopped.value.code !== 0) { yield* Effect.logWarning("Could not stop agents left by a previous server", { slice: ownedSlice, stderr: stopped._tag === "Some" ? stopped.value.stderr.trim() : undefined, }); } }); const systemd = yield* Effect.cached( Effect.gen(function* () { const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR; if (runtimeDir === undefined || runtimeDir.length === 0) return undefined; const systemdRun = yield* resolve("systemd-run", hostEnvironment); const systemctl = yield* resolve("systemctl", hostEnvironment); if (systemdRun === null || systemctl === null) return undefined; yield* stopLeftoverAgents(systemctl); const probe = (options: ReadonlyArray<string>) => run(systemdRun, [ "--user", "--scope", "--quiet", "--collect", `--slice=${AGENT_SLICE}`, ...options, "--", "true", ]).pipe(Effect.map((result) => result._tag === "Some" && result.value.code === 0)); // Scopes take OOMPolicy from systemd 253; older managers still get scopes. const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, "--property=OOMPolicy=stop"]); if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) { yield* Effect.logInfo("Agent scopes are off: no systemd user manager"); return undefined; } const setProperties = Effect.fn("agentScope.setProperties")(function* ( slice: string, properties: ReadonlyArray<string>, ) { const result = yield* run(systemctl, [ "--user", "set-property", "--runtime", slice, ...properties, ]); if (result._tag === "None" || result.value.code !== 0) { yield* Effect.logWarning("Could not set resource limits on an agents slice", { slice, properties, stderr: result._tag === "Some" ? result.value.stderr.trim() : undefined, }); } }); const limits = agentSliceMemoryLimits(NodeOS.totalmem()); yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]); yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES); yield* Effect.logInfo("Agent scopes are on", { slice: AGENT_SLICE, ...limits, oomPolicy: supportsOomPolicy, }); return { systemdRun, systemctl, runtimeDir, supportsOomPolicy, }; }), ); // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd. const scopeByThread = new Map<string, { unit: string; oomKilled: boolean; shared: boolean }>(); const readScope = Effect.fn("agentScope.readScope")(function* (systemctl: string, unit: string) { const shown = yield* run(systemctl, [ "--user", "show", unit, "--property=LoadState,ActiveState,Result,ControlGroup", ]); if (shown._tag === "None" || shown.value.code !== 0) return undefined; const values = parseKeyValues(shown.value.stdout); const controlGroup = values.get("ControlGroup") ?? ""; const memoryEvents = controlGroup ? yield* readCgroupFile(controlGroup, "memory.events") : ""; const cgroupEvents = controlGroup ? yield* readCgroupFile(controlGroup, "cgroup.events") : ""; const populated = parseCgroupCounter(cgroupEvents, "populated"); return { loadState: values.get("LoadState") ?? "", activeState: values.get("ActiveState") ?? "", result: values.get("Result") ?? "", oomKills: parseCgroupCounter(memoryEvents, "oom_kill"), populated: populated === undefined ? undefined : populated === 1, } satisfies ScopeState; }); const clearFailed = Effect.fn("agentScope.clearFailed")(function* ( systemctl: string, unit: string, ) { for (let attempt = 0; attempt < CLEAR_FAILED_ATTEMPTS; attempt++) { const state = yield* readScope(systemctl, unit); if (state === undefined) return; if (state.activeState === "failed") { yield* run(systemctl, ["--user", "reset-failed", unit]); return; } if (state.activeState !== "deactivating") return; yield* Effect.sleep(CLEAR_FAILED_INTERVAL); } }); const forget = (systemctl: string, unit: string) => clearFailed(systemctl, unit).pipe(Effect.forkIn(layerScope), Effect.asVoid); const evictOldest = Effect.fn("agentScope.evictOldest")(function* (systemctl: string) { const oldest = scopeByThread.entries().next(); if (oldest.done === true) return; const [threadId, tracked] = oldest.value; const state = yield* readScope(systemctl, tracked.unit); if (scopeByThread.get(threadId) !== tracked) return; scopeByThread.delete(threadId); const status = state === undefined ? "running" : classifyScope(state); if (status === "running" || status === "stopping") { scopeByThread.set(threadId, tracked); return; } yield* forget(systemctl, tracked.unit); }); const track = Effect.fn("agentScope.track")(function* ( systemctl: string, threadId: string, unit: string | undefined, shared = false, ) { const previous = scopeByThread.get(threadId); scopeByThread.delete(threadId); if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared }); if (previous !== undefined) yield* forget(systemctl, previous.unit); if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl); }); const oomKilled: AgentScopeShape["oomKilled"] = Effect.fn("agentScope.oomKilled")( function* (threadId) { const tracked = scopeByThread.get(threadId); if (tracked === undefined) return false; if (tracked.oomKilled) return true; const scope = yield* systemd; if (scope === undefined) return false; for (let attempt = 0; attempt < STOPPING_POLL_ATTEMPTS; attempt++) { const state = yield* readScope(scope.systemctl, tracked.unit); if (state === undefined) return false; const status = classifyScope(state); if (status === "running") return false; if (status === "stopping") { yield* Effect.sleep(STOPPING_POLL_INTERVAL); continue; } yield* forget(scope.systemctl, tracked.unit); if (status === "gone") return false; yield* Effect.logWarning("Agent scope was killed: out of memory", { threadId, unit: tracked.unit, }); tracked.oomKilled = true; return true; } return false; }, ); const stop = Effect.fn("AgentScope.stop")(function* (unit: string) { if (!unit.startsWith("t3code-") || !unit.endsWith(".scope")) return; const scope = yield* systemd; if (scope === undefined) return; const state = yield* readScope(scope.systemctl, unit); if ( state === undefined || state.loadState !== "loaded" || state.activeState !== "active" || state.result === "oom-kill" || state.populated === false ) return; // The kernel counts OOM kills before signaling the victim. Let OOMPolicy // record the result before a manual stop can overwrite it; an unreadable // counter cannot rule out that race. Older managers need us to stop it. if (scope.supportsOomPolicy && (state.oomKills === undefined || state.oomKills > 0)) return; const stopped = yield* run(scope.systemctl, ["--user", "stop", "--no-block", unit]); if (stopped._tag === "None" || stopped.value.code !== 0) { yield* Effect.logWarning("Could not stop an exited agent's scope", { unit, exitCode: stopped._tag === "Some" ? stopped.value.code : undefined, }); } }); const kill: AgentScopeShape["kill"] = Effect.fn("AgentScope.kill")(function* (threadId) { const tracked = scopeByThread.get(threadId); if (tracked === undefined || tracked.shared) return false; const scope = yield* systemd; if (scope === undefined) return false; const shown = yield* run(scope.systemctl, [ "--user", "show", tracked.unit, "--property=ActiveState,ControlGroup", ]); if (shown._tag === "None" || shown.value.code !== 0) return false; const values = parseKeyValues(shown.value.stdout); if (values.get("ActiveState") !== "active") return false; const controlGroup = values.get("ControlGroup") ?? ""; // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback. const killedByCgroup = controlGroup.length > 0 && (yield* fileSystem .writeFileString(path.join("/sys/fs/cgroup", controlGroup, "cgroup.kill"), "1") .pipe( Effect.as(true), Effect.orElseSucceed(() => false), )); if (killedByCgroup) return true; const killed = yield* run(scope.systemctl, [ "--user", "kill", "--signal=SIGKILL", tracked.unit, ]); return killed._tag === "Some" && killed.value.code === 0; }); const clear: AgentScopeShape["clear"] = Effect.fn("agentScope.clear")(function* (threadId) { if (!scopeByThread.has(threadId)) return; const scope = yield* systemd; if (scope !== undefined) yield* track(scope.systemctl, threadId, undefined); }); return { wrap: Effect.fn("AgentScope.wrap")(function* ({ command, args, name, threadId, env, survivesRestart, shared, }) { const resolved = yield* resolve(command, env ?? hostEnvironment); if (resolved === null) return { command, args }; const scope = yield* systemd; if (scope === undefined) return agentScopeCommand({ command: resolved, args }); const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`; // A continuing shared scope cannot attribute a tool's OOM to one thread. // Clear the launch thread's previous scope instead. const continuing = shared === true && scope.supportsOomPolicy; const owned = threadId !== undefined && !continuing; if (threadId !== undefined) yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true); const systemdScope: AgentSystemdScope = { systemdRun: scope.systemdRun, runtimeDir: scope.runtimeDir, slice: survivesRestart === true ? AGENT_SLICE : ownedSlice, unit, // Without an owning thread nothing reads the result, so let systemd // unload the unit even when it fails. options: [ ...SCOPE_OPTIONS, ...(scope.supportsOomPolicy ? [`--property=OOMPolicy=${continuing ? "continue" : "stop"}`] : []), ...(owned ? [] : ["--collect"]), ], }; return { ...agentScopeCommand({ command: resolved, args, scope: systemdScope }), unit }; }), oomKilled, stop, kill, clear, } satisfies AgentScopeShape; }); export const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer));
apps/server/src/process/agentScope.ts at main · sailfork/t3code-github · Sailfork
, \"m\").exec(text);\n return match === null ? undefined : Number(match[1]);\n}\n\n// systemd's default 90 s stop timeout would outlast our leftover-agent wait.\n// Delegation lets rootless containers (Podman, kind) manage their own cgroups.\nconst SCOPE_OPTIONS = [\"--property=TimeoutStopSec=10s\", \"--property=Delegate=yes\"];\nconst STOPPING_POLL_INTERVAL = \"100 millis\";\nconst STOPPING_POLL_ATTEMPTS = 20;\nconst CLEAR_FAILED_INTERVAL = \"1 second\";\nconst CLEAR_FAILED_ATTEMPTS = 120;\nconst MAX_TRACKED_THREADS = 500;\n\nexport const make = Effect.gen(function* () {\n const platform = yield* HostProcess.Platform;\n if (platform !== \"linux\") return AgentScope.defaultValue();\n\n const runner = yield* ProcessRunner.ProcessRunner;\n const fileSystem = yield* FileSystem.FileSystem;\n const path = yield* Path.Path;\n const hostEnvironment = yield* HostProcess.Environment;\n const { stateDir } = yield* ServerConfig.ServerConfig;\n const ownedSlice = ownedAgentSlice(stateDir);\n const layerScope = yield* Effect.scope;\n\n const resolve = (command: string, env: NodeJS.ProcessEnv) =>\n resolveCommandPath(command, { env }).pipe(\n Effect.provideService(FileSystem.FileSystem, fileSystem),\n Effect.provideService(Path.Path, path),\n Effect.provideService(HostProcess.Platform, platform),\n );\n\n const run = (command: string, args: ReadonlyArray\u003cstring>) =>\n runner.run({ command, args, timeout: \"5 seconds\" }).pipe(Effect.option);\n\n const readCgroupFile = (controlGroup: string, file: string) =>\n fileSystem\n .readFileString(path.join(\"/sys/fs/cgroup\", controlGroup, file))\n .pipe(Effect.orElseSucceed(() => \"\"));\n\n // Old agents keep their scopes when the server is killed, and the restarted\n // server resumes their sessions. Stop them first, so two copies of one\n // session never work in the same worktree.\n const stopLeftoverAgents = Effect.fn(\"agentScope.stopLeftoverAgents\")(function* (\n systemctl: string,\n ) {\n const shown = yield* run(systemctl, [\n \"--user\",\n \"show\",\n ownedSlice,\n \"--property=ActiveState,ControlGroup\",\n ]);\n if (shown._tag === \"None\") return;\n const values = parseKeyValues(shown.value.stdout);\n if (values.get(\"ActiveState\") !== \"active\") return;\n const controlGroup = values.get(\"ControlGroup\") ?? \"\";\n const events = controlGroup ? yield* readCgroupFile(controlGroup, \"cgroup.events\") : \"\";\n if (parseCgroupCounter(events, \"populated\") === 0) return;\n yield* Effect.logWarning(\"Stopping agents left by a previous server\", { slice: ownedSlice });\n const stopped = yield* runner\n .run({ command: systemctl, args: [\"--user\", \"stop\", ownedSlice], timeout: \"30 seconds\" })\n .pipe(Effect.option);\n if (stopped._tag === \"None\" || stopped.value.code !== 0) {\n yield* Effect.logWarning(\"Could not stop agents left by a previous server\", {\n slice: ownedSlice,\n stderr: stopped._tag === \"Some\" ? stopped.value.stderr.trim() : undefined,\n });\n }\n });\n\n const systemd = yield* Effect.cached(\n Effect.gen(function* () {\n const runtimeDir = hostEnvironment.XDG_RUNTIME_DIR;\n if (runtimeDir === undefined || runtimeDir.length === 0) return undefined;\n const systemdRun = yield* resolve(\"systemd-run\", hostEnvironment);\n const systemctl = yield* resolve(\"systemctl\", hostEnvironment);\n if (systemdRun === null || systemctl === null) return undefined;\n yield* stopLeftoverAgents(systemctl);\n const probe = (options: ReadonlyArray\u003cstring>) =>\n run(systemdRun, [\n \"--user\",\n \"--scope\",\n \"--quiet\",\n \"--collect\",\n `--slice=${AGENT_SLICE}`,\n ...options,\n \"--\",\n \"true\",\n ]).pipe(Effect.map((result) => result._tag === \"Some\" && result.value.code === 0));\n // Scopes take OOMPolicy from systemd 253; older managers still get scopes.\n const supportsOomPolicy = yield* probe([...SCOPE_OPTIONS, \"--property=OOMPolicy=stop\"]);\n if (!supportsOomPolicy && !(yield* probe(SCOPE_OPTIONS))) {\n yield* Effect.logInfo(\"Agent scopes are off: no systemd user manager\");\n return undefined;\n }\n const setProperties = Effect.fn(\"agentScope.setProperties\")(function* (\n slice: string,\n properties: ReadonlyArray\u003cstring>,\n ) {\n const result = yield* run(systemctl, [\n \"--user\",\n \"set-property\",\n \"--runtime\",\n slice,\n ...properties,\n ]);\n if (result._tag === \"None\" || result.value.code !== 0) {\n yield* Effect.logWarning(\"Could not set resource limits on an agents slice\", {\n slice,\n properties,\n stderr: result._tag === \"Some\" ? result.value.stderr.trim() : undefined,\n });\n }\n });\n const limits = agentSliceMemoryLimits(NodeOS.totalmem());\n yield* setProperties(AGENT_SLICE, [`MemoryHigh=${limits.high}`, `MemoryMax=${limits.max}`]);\n yield* setProperties(AGENTS_PARENT_SLICE, AGENTS_PARENT_PROPERTIES);\n yield* Effect.logInfo(\"Agent scopes are on\", {\n slice: AGENT_SLICE,\n ...limits,\n oomPolicy: supportsOomPolicy,\n });\n return {\n systemdRun,\n systemctl,\n runtimeDir,\n supportsOomPolicy,\n };\n }),\n );\n\n // `shared` scopes serve other threads too: tracked only for OOM diagnosis on old systemd.\n const scopeByThread = new Map\u003cstring, { unit: string; oomKilled: boolean; shared: boolean }>();\n\n const readScope = Effect.fn(\"agentScope.readScope\")(function* (systemctl: string, unit: string) {\n const shown = yield* run(systemctl, [\n \"--user\",\n \"show\",\n unit,\n \"--property=LoadState,ActiveState,Result,ControlGroup\",\n ]);\n if (shown._tag === \"None\" || shown.value.code !== 0) return undefined;\n const values = parseKeyValues(shown.value.stdout);\n const controlGroup = values.get(\"ControlGroup\") ?? \"\";\n const memoryEvents = controlGroup ? yield* readCgroupFile(controlGroup, \"memory.events\") : \"\";\n const cgroupEvents = controlGroup ? yield* readCgroupFile(controlGroup, \"cgroup.events\") : \"\";\n const populated = parseCgroupCounter(cgroupEvents, \"populated\");\n return {\n loadState: values.get(\"LoadState\") ?? \"\",\n activeState: values.get(\"ActiveState\") ?? \"\",\n result: values.get(\"Result\") ?? \"\",\n oomKills: parseCgroupCounter(memoryEvents, \"oom_kill\"),\n populated: populated === undefined ? undefined : populated === 1,\n } satisfies ScopeState;\n });\n\n const clearFailed = Effect.fn(\"agentScope.clearFailed\")(function* (\n systemctl: string,\n unit: string,\n ) {\n for (let attempt = 0; attempt \u003c CLEAR_FAILED_ATTEMPTS; attempt++) {\n const state = yield* readScope(systemctl, unit);\n if (state === undefined) return;\n if (state.activeState === \"failed\") {\n yield* run(systemctl, [\"--user\", \"reset-failed\", unit]);\n return;\n }\n if (state.activeState !== \"deactivating\") return;\n yield* Effect.sleep(CLEAR_FAILED_INTERVAL);\n }\n });\n\n const forget = (systemctl: string, unit: string) =>\n clearFailed(systemctl, unit).pipe(Effect.forkIn(layerScope), Effect.asVoid);\n\n const evictOldest = Effect.fn(\"agentScope.evictOldest\")(function* (systemctl: string) {\n const oldest = scopeByThread.entries().next();\n if (oldest.done === true) return;\n const [threadId, tracked] = oldest.value;\n const state = yield* readScope(systemctl, tracked.unit);\n if (scopeByThread.get(threadId) !== tracked) return;\n scopeByThread.delete(threadId);\n const status = state === undefined ? \"running\" : classifyScope(state);\n if (status === \"running\" || status === \"stopping\") {\n scopeByThread.set(threadId, tracked);\n return;\n }\n yield* forget(systemctl, tracked.unit);\n });\n\n const track = Effect.fn(\"agentScope.track\")(function* (\n systemctl: string,\n threadId: string,\n unit: string | undefined,\n shared = false,\n ) {\n const previous = scopeByThread.get(threadId);\n scopeByThread.delete(threadId);\n if (unit !== undefined) scopeByThread.set(threadId, { unit, oomKilled: false, shared });\n if (previous !== undefined) yield* forget(systemctl, previous.unit);\n if (scopeByThread.size > MAX_TRACKED_THREADS) yield* evictOldest(systemctl);\n });\n\n const oomKilled: AgentScopeShape[\"oomKilled\"] = Effect.fn(\"agentScope.oomKilled\")(\n function* (threadId) {\n const tracked = scopeByThread.get(threadId);\n if (tracked === undefined) return false;\n if (tracked.oomKilled) return true;\n const scope = yield* systemd;\n if (scope === undefined) return false;\n for (let attempt = 0; attempt \u003c STOPPING_POLL_ATTEMPTS; attempt++) {\n const state = yield* readScope(scope.systemctl, tracked.unit);\n if (state === undefined) return false;\n const status = classifyScope(state);\n if (status === \"running\") return false;\n if (status === \"stopping\") {\n yield* Effect.sleep(STOPPING_POLL_INTERVAL);\n continue;\n }\n yield* forget(scope.systemctl, tracked.unit);\n if (status === \"gone\") return false;\n yield* Effect.logWarning(\"Agent scope was killed: out of memory\", {\n threadId,\n unit: tracked.unit,\n });\n tracked.oomKilled = true;\n return true;\n }\n return false;\n },\n );\n\n const stop = Effect.fn(\"AgentScope.stop\")(function* (unit: string) {\n if (!unit.startsWith(\"t3code-\") || !unit.endsWith(\".scope\")) return;\n const scope = yield* systemd;\n if (scope === undefined) return;\n const state = yield* readScope(scope.systemctl, unit);\n if (\n state === undefined ||\n state.loadState !== \"loaded\" ||\n state.activeState !== \"active\" ||\n state.result === \"oom-kill\" ||\n state.populated === false\n )\n return;\n // The kernel counts OOM kills before signaling the victim. Let OOMPolicy\n // record the result before a manual stop can overwrite it; an unreadable\n // counter cannot rule out that race. Older managers need us to stop it.\n if (scope.supportsOomPolicy && (state.oomKills === undefined || state.oomKills > 0)) return;\n const stopped = yield* run(scope.systemctl, [\"--user\", \"stop\", \"--no-block\", unit]);\n if (stopped._tag === \"None\" || stopped.value.code !== 0) {\n yield* Effect.logWarning(\"Could not stop an exited agent's scope\", {\n unit,\n exitCode: stopped._tag === \"Some\" ? stopped.value.code : undefined,\n });\n }\n });\n\n const kill: AgentScopeShape[\"kill\"] = Effect.fn(\"AgentScope.kill\")(function* (threadId) {\n const tracked = scopeByThread.get(threadId);\n if (tracked === undefined || tracked.shared) return false;\n const scope = yield* systemd;\n if (scope === undefined) return false;\n const shown = yield* run(scope.systemctl, [\n \"--user\",\n \"show\",\n tracked.unit,\n \"--property=ActiveState,ControlGroup\",\n ]);\n if (shown._tag === \"None\" || shown.value.code !== 0) return false;\n const values = parseKeyValues(shown.value.stdout);\n if (values.get(\"ActiveState\") !== \"active\") return false;\n const controlGroup = values.get(\"ControlGroup\") ?? \"\";\n // cgroup.kill (Linux 5.14+) kills the whole tree at once; systemd's kill is the fallback.\n const killedByCgroup =\n controlGroup.length > 0 &&\n (yield* fileSystem\n .writeFileString(path.join(\"/sys/fs/cgroup\", controlGroup, \"cgroup.kill\"), \"1\")\n .pipe(\n Effect.as(true),\n Effect.orElseSucceed(() => false),\n ));\n if (killedByCgroup) return true;\n const killed = yield* run(scope.systemctl, [\n \"--user\",\n \"kill\",\n \"--signal=SIGKILL\",\n tracked.unit,\n ]);\n return killed._tag === \"Some\" && killed.value.code === 0;\n });\n\n const clear: AgentScopeShape[\"clear\"] = Effect.fn(\"agentScope.clear\")(function* (threadId) {\n if (!scopeByThread.has(threadId)) return;\n const scope = yield* systemd;\n if (scope !== undefined) yield* track(scope.systemctl, threadId, undefined);\n });\n\n return {\n wrap: Effect.fn(\"AgentScope.wrap\")(function* ({\n command,\n args,\n name,\n threadId,\n env,\n survivesRestart,\n shared,\n }) {\n const resolved = yield* resolve(command, env ?? hostEnvironment);\n if (resolved === null) return { command, args };\n const scope = yield* systemd;\n if (scope === undefined) return agentScopeCommand({ command: resolved, args });\n const unit = `t3code-${name}-${NodeCrypto.randomUUID().slice(0, 8)}.scope`;\n // A continuing shared scope cannot attribute a tool's OOM to one thread.\n // Clear the launch thread's previous scope instead.\n const continuing = shared === true && scope.supportsOomPolicy;\n const owned = threadId !== undefined && !continuing;\n if (threadId !== undefined)\n yield* track(scope.systemctl, threadId, owned ? unit : undefined, shared === true);\n const systemdScope: AgentSystemdScope = {\n systemdRun: scope.systemdRun,\n runtimeDir: scope.runtimeDir,\n slice: survivesRestart === true ? AGENT_SLICE : ownedSlice,\n unit,\n // Without an owning thread nothing reads the result, so let systemd\n // unload the unit even when it fails.\n options: [\n ...SCOPE_OPTIONS,\n ...(scope.supportsOomPolicy\n ? [`--property=OOMPolicy=${continuing ? \"continue\" : \"stop\"}`]\n : []),\n ...(owned ? [] : [\"--collect\"]),\n ],\n };\n return { ...agentScopeCommand({ command: resolved, args, scope: systemdScope }), unit };\n }),\n oomKilled,\n stop,\n kill,\n clear,\n } satisfies AgentScopeShape;\n});\n\nexport const layer = Layer.effect(AgentScope, make).pipe(Layer.provide(ProcessRunner.layer));\n","preview":null,"history":null,"version":"b9b6b56acb140b473a867f80b0c70e5439d193df"},"/_session?shell=1":null}