import { assert, it } from "@effect/vitest";
import {
CommandId,
ContextTransferId,
EventId,
MessageId,
NodeId,
type OrchestrationV2AppThread,
type OrchestrationV2Run,
ProjectId,
ProviderDriverKind,
ProviderInstanceId,
ProviderSessionId,
ProviderThreadId,
ProviderTurnId,
RunId,
RuntimeRequestId,
ThreadId,
TurnItemId,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Stream from "effect/Stream";
import * as SqlClient from "effect/sql/SqlClient";
import * as Statement from "effect/sql/Statement";
import delegatedRecoveryIndexes from "../persistence/Migrations/079_DelegatedRecoveryIndexes.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as ServerSettings from "../serverSettings.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as EventSink from "./EventSink.ts";
import * as EventStore from "./EventStore.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
import { restartContinuationRun } from "./RestartContinuation.ts";
const layerTest = Layer.mergeAll(ProjectionStore.layer, EffectOutbox.layer).pipe(
Layer.provideMerge(SqlitePersistence.layerMemory),
);
const layerRecoveryTest = ProviderRuntimeRecovery.layer.pipe(
Layer.provideMerge(
EventSink.layer.pipe(
Layer.provideMerge(
Layer.mergeAll(
EventStore.layer,
ProjectionStore.layer,
EffectOutbox.layer,
IdAllocator.layer,
).pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),
),
),
),
Layer.provide(ServerSettings.layerTest()),
);
const providerInstanceId = ProviderInstanceId.make("codex");
const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
const driver = ProviderDriverKind.make("codex");
const createThread = Effect.fn(function* (
name: string,
overrides: Partial<OrchestrationV2AppThread> = {},
) {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make(`thread:recovery:${name}`);
const thread: OrchestrationV2AppThread = {
createdBy: "user",
creationSource: "web",
id: threadId,
projectId: ProjectId.make("project:recovery"),
title: name,
providerInstanceId,
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
activeProviderThreadId: null,
lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
forkedFrom: null,
createdAt: now,
updatedAt: now,
archivedAt: null,
settledOverride: null,
settledAt: null,
lastVisitedAt: null,
deletedAt: null,
...overrides,
};
yield* projections.apply({
id: EventId.make(`event:${threadId}:created`),
type: "thread.created",
threadId,
occurredAt: now,
payload: thread,
});
return threadId;
});
const createRun = Effect.fn(function* (
threadId: ThreadId,
status: OrchestrationV2Run["status"],
overrides: Partial<OrchestrationV2Run> = {},
) {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const ordinal = overrides.ordinal ?? 1;
const runId = RunId.make(`run:${threadId}:${ordinal}`);
yield* projections.apply({
id: EventId.make(`event:${runId}:created`),
type: "run.created",
threadId,
runId,
occurredAt: now,
payload: {
id: runId,
threadId,
ordinal,
providerInstanceId,
modelSelection,
providerThreadId: null,
userMessageId: MessageId.make(`message:${runId}`),
rootNodeId: null,
activeAttemptId: null,
status,
requestedAt: now,
startedAt: now,
completedAt: status === "completed" ? now : null,
checkpointId: null,
contextHandoffId: null,
...overrides,
},
});
return runId;
});
it.effect.each(["startup", "shutdown"] as const)(
"%s cancels queued buffered wakes and holds durable messages",
(trigger) =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const now = yield* DateTime.now;
const bufferedThreads = [] as ThreadId[];
const durableThreads = [] as ThreadId[];
for (const kind of ["buffered", "user", "notification", "delegated"] as const) {
const threadId = yield* createThread(`wake:${trigger}:${kind}`);
const buffered = kind === "buffered";
(buffered ? bufferedThreads : durableThreads).push(threadId);
const messageId = MessageId.make(`message:${threadId}`);
const runId = yield* createRun(threadId, "queued", {
userMessageId: messageId,
startedAt: null,
queuePosition: 1,
});
yield* createRun(threadId, "completed", { ordinal: 2 });
yield* projections.apply({
id: EventId.make(`message:${threadId}`),
type: "message.updated",
threadId,
runId,
occurredAt: now,
payload: {
id: messageId,
threadId,
runId,
nodeId: null,
role: "user",
text: "Queued work",
attachments: [],
streaming: false,
createdAt: now,
updatedAt: now,
createdBy: kind === "user" ? "user" : "agent",
creationSource: buffered ? "provider" : "server",
...(kind === "user" || kind === "delegated"
? {}
: {
notification: {
source: { kind: "background_task" as const },
outcome: "updated" as const,
summary: "Background activity updated",
},
}),
...(kind === "delegated"
? { delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [] } }
: {}),
},
});
}
assert.deepEqual(
new Set(yield* projections.getRecoveryThreadIds("runtime")),
new Set([...bufferedThreads, ...durableThreads]),
);
const summary = yield* recovery.reconcile(trigger);
assert.equal(summary.terminalizedRuns, 1);
for (const threadId of bufferedThreads) {
const projection = yield* projections.getThreadProjection(threadId);
assert.equal(projection.runs[0]?.status, "cancelled");
assert.isNotNull(projection.runs[0]?.completedAt);
assert.isNull(projection.runs[0]?.queuePosition);
assert.equal(projection.runs[1]?.status, "completed");
}
for (const threadId of durableThreads) {
const projection = yield* projections.getThreadProjection(threadId);
assert.equal(projection.runs[0]?.status, "queued");
assert.isTrue(projection.runs[0]?.queueHeld);
}
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
}).pipe(Effect.provide(layerRecoveryTest)),
);
it.effect.each(["startup", "shutdown"] as const)(
"stops a detached session and its orphaned active provider thread on %s",
(trigger) =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const now = yield* DateTime.now;
const threadId = yield* createThread("detached-session");
const providerSessionId = ProviderSessionId.make("session:recovery:detached");
yield* projections.apply({
id: EventId.make("event:recovery:detached-session:attached"),
type: "provider-session.attached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerSessionId,
driver,
providerInstanceId,
status: "ready",
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
},
});
yield* projections.apply({
id: EventId.make("event:recovery:detached-session:detached"),
type: "provider-session.detached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
});
const providerThreadId = ProviderThreadId.make("provider-thread:recovery:detached");
yield* projections.apply({
id: EventId.make("event:recovery:detached-provider-thread"),
type: "provider-thread.updated",
threadId,
occurredAt: now,
payload: {
id: providerThreadId,
appThreadId: threadId,
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: "active",
firstRunOrdinal: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
},
});
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), [threadId]);
const summary = yield* recovery.reconcile(trigger);
assert.equal(summary.stoppedSessions, 1);
const sql = yield* SqlClient.SqlClient;
const rows = yield* sql<{ readonly status: string }>`
SELECT status FROM orchestration_v2_projection_provider_sessions
WHERE provider_session_id = ${providerSessionId}
`;
assert.deepEqual(rows, [{ status: "stopped" }]);
const projection = yield* projections.getThreadProjection(threadId);
assert.equal(projection.providerThreads[0]?.status, "idle");
const eventStore = yield* EventStore.EventStoreV2;
const sequence = yield* eventStore.latestSequence();
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
yield* recovery.reconcile(trigger);
assert.equal(yield* eventStore.latestSequence(), sequence);
}).pipe(Effect.provide(layerRecoveryTest)),
);
it.effect.each(["attached", "detached", "replaced"] as const)(
"recovers attached sessions without restoring detached errors, session=%s",
(state) =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const now = yield* DateTime.now;
const threadId = yield* createThread("failed-status-session");
const providerSessionId = ProviderSessionId.make("session:recovery:failed-status");
const session = {
id: providerSessionId,
driver,
providerInstanceId,
status: "error" as const,
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
extensionStatuses: [{ key: "watch", text: "Watching" }],
createdAt: now,
updatedAt: now,
lastError: "Rate limited",
};
yield* projections.apply({
id: EventId.make("event:recovery:failed-status:attached"),
type: "provider-session.attached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: session,
});
if (state !== "attached") {
yield* projections.apply({
id: EventId.make("event:recovery:failed-status:detached"),
type: "provider-session.detached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: { providerSessionId, detachedAt: now, reason: "Thread settled." },
});
}
const replacementSessionId = ProviderSessionId.make("session:recovery:replacement");
if (state === "replaced") {
yield* projections.apply({
id: EventId.make("event:recovery:replacement:attached"),
type: "provider-session.attached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
...session,
id: replacementSessionId,
status: "ready",
extensionStatuses: [],
lastError: null,
},
});
yield* createRun(threadId, "completed");
}
assert.deepEqual(
yield* projections.getRecoveryThreadIds("runtime"),
state === "detached" ? [] : [threadId],
);
yield* recovery.reconcile("startup");
const eventStore = yield* EventStore.EventStoreV2;
const stored = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
const sessionUpdates = stored.flatMap(({ event }) =>
event.type === "provider-session.updated"
? [
{
id: event.payload.id,
status: event.payload.status,
lastError: event.payload.lastError,
extensionStatuses: event.payload.extensionStatuses,
},
]
: [],
);
assert.deepEqual(
sessionUpdates,
state === "detached"
? []
: [
{
id: state === "attached" ? providerSessionId : replacementSessionId,
status: state === "attached" ? "error" : "stopped",
lastError: state === "attached" ? "Rate limited" : null,
extensionStatuses: [],
},
],
);
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
}).pipe(Effect.provide(layerRecoveryTest)),
);
it.effect("selects unfinished recovery work without reading settled thread histories", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
for (let index = 0; index < 600; index += 1) {
yield* createRun(yield* createThread(`settled-${index}`), "completed");
}
const queued = yield* createThread("queued");
const archived = yield* createThread("archived", { archivedAt: now });
const deleted = yield* createThread("deleted", { deletedAt: now });
const blocked = yield* createThread("blocked");
yield* createRun(queued, "completed");
for (const threadId of [queued, archived, deleted, blocked]) {
yield* createRun(threadId, "queued", threadId === queued ? { ordinal: 2 } : {});
}
yield* createRun(blocked, "waiting", { ordinal: 2 });
const background = yield* createThread("background");
yield* createRun(background, "completed");
yield* projections.apply({
id: EventId.make("event:recovery:background"),
type: "turn-item.updated",
threadId: background,
occurredAt: now,
payload: {
id: TurnItemId.make("item:recovery:background"),
threadId: background,
runId: null,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 1,
type: "dynamic_tool",
status: "waiting",
title: null,
startedAt: now,
completedAt: null,
updatedAt: now,
toolName: "background",
input: null,
output: null,
},
});
const outboxOnly = yield* createThread("outbox-only");
yield* outbox.enqueue([
{
id: "effect:recovery:outbox-only",
commandId: CommandId.make("command:recovery:outbox-only"),
threadId: outboxOnly,
request: { type: "provider-turn.start", runId: RunId.make("run:outbox-only") },
},
]);
const requestOnly = yield* createThread("request-only");
yield* projections.apply({
id: EventId.make("event:recovery:request-only"),
type: "runtime-request.updated",
threadId: requestOnly,
occurredAt: now,
payload: {
id: RuntimeRequestId.make("request:recovery:pending"),
nodeId: NodeId.make("node:recovery:request"),
providerTurnId: null,
nativeRequestRef: null,
kind: "user_input",
status: "pending",
responseCapability: { type: "not_resumable", reason: "Process stopped" },
createdAt: now,
resolvedAt: null,
},
});
const delivery = yield* createThread("delivery", { archivedAt: now });
yield* createRun(delivery, "completed", {
delegatedCompletion: {
disposition: "open",
nextGeneration: 2,
delivery: { generation: 1, messageId: MessageId.make("message:delivery"), taskIds: [] },
},
});
yield* sql`
UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
WHERE thread_id = ${ThreadId.make("thread:recovery:settled-599")}
`;
yield* sql`
UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
WHERE thread_id = ${queued} AND ordinal = 1
`;
assert.deepEqual(yield* projections.getRecoveryThreadIds("queued-runs"), [queued]);
assert.deepEqual(
new Set(yield* projections.getRecoveryThreadIds("runtime")),
new Set([queued, archived, blocked, background, outboxOnly, requestOnly]),
);
for (const [kind, expected, index] of [
["delegated-completions", [delivery], "runs_delegated_delivery_idx"],
["subagent-results", [], "threads_subagent_result_idx"],
] as const) {
const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
const observe: Statement.Transformer = (statement) => {
statements.push(statement.compile());
return Effect.succeed(statement);
};
assert.deepEqual(
yield* projections
.getRecoveryThreadIds(kind)
.pipe(Effect.provideService(Statement.CurrentTransformer, observe)),
expected,
);
const [query, params] = statements[0]!;
const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
assert.isTrue(
plan.some((row) => row.detail.includes(index)),
JSON.stringify(plan),
);
}
const recoveryState = yield* projections.getRuntimeRecoveryProjection(queued);
assert.deepEqual(
recoveryState.runs.map((run) => run.id),
[RunId.make(`run:${queued}:2`)],
);
assert.deepEqual(yield* projections.getUnreadableThreadIds(), [
queued,
ThreadId.make("thread:recovery:settled-599"),
]);
}).pipe(Effect.provide(layerTest)),
);
it.effect("maintains delegated candidate indexes across legacy rows and payload transitions", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const parent = yield* createThread("index-parent");
const child = yield* createThread("index-child");
const run = yield* createRun(child, "completed");
const rows = yield* sql<{ payload_json: string }>`
SELECT payload_json FROM orchestration_v2_projection_threads WHERE thread_id = ${child}
`;
const original = JSON.parse(rows[0]!.payload_json);
for (const payload of [
"{broken",
JSON.stringify({
...original,
lineage: { parentThreadId: parent, relationshipToParent: "subagent" },
forkedFrom: { type: "provider_thread" },
}),
JSON.stringify({
...original,
lineage: { parentThreadId: parent, relationshipToParent: "subagent" },
forkedFrom: { type: "node", nodeId: "node:index" },
}),
JSON.stringify({
...original,
lineage: { parentThreadId: child, relationshipToParent: "subagent" },
forkedFrom: { type: "node", nodeId: "node:index" },
}),
JSON.stringify(original),
]) {
yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = ${payload}
WHERE thread_id = ${child}`;
const expected = payload.includes('"type":"node"') ? [child] : [];
assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), expected);
}
for (const entry of [
["{broken", false],
["{}", false],
['{"delegatedCompletion":{"delivery":null}}', false],
['{"delegatedCompletion":{"delivery":[]}}', false],
['{"delegatedCompletion":{"delivery":{}}}', true],
["{}", false],
] as const) {
const [payload, included] = entry;
yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = ${payload}
WHERE run_id = ${run}`;
assert.deepEqual(
yield* projections.getRecoveryThreadIds("delegated-completions"),
included ? [child] : [],
);
}
yield* sql`UPDATE orchestration_v2_projection_threads SET payload_json = '{broken'
WHERE thread_id = ${parent}`;
yield* sql`UPDATE orchestration_v2_projection_runs SET payload_json = '{broken'
WHERE run_id = ${run}`;
const existing = yield* createThread("index-existing", {
lineage: { parentThreadId: child, relationshipToParent: "subagent", rootThreadId: parent },
forkedFrom: { type: "node", nodeId: NodeId.make("node:index-existing") },
});
yield* createRun(existing, "completed", {
delegatedCompletion: {
disposition: "open",
nextGeneration: 2,
delivery: {
generation: 1,
messageId: MessageId.make("message:index-existing"),
taskIds: [],
},
},
});
yield* sql`DROP INDEX orchestration_v2_projection_threads_subagent_result_idx`;
yield* sql`DROP INDEX orchestration_v2_projection_runs_delegated_delivery_idx`;
yield* delegatedRecoveryIndexes;
assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [existing]);
assert.deepEqual(yield* projections.getRecoveryThreadIds("delegated-completions"), [existing]);
}).pipe(Effect.provide(layerTest)),
);
it.effect("recovers terminal subagent results until their cross-thread transfer exists", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const parent = yield* createThread("subagent-parent");
const children: Array<ThreadId> = [];
for (const name of ["terminal", "archived", "deleted", "running", "held", "queued"]) {
const child = yield* createThread(`subagent-${name}`, {
lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent },
forkedFrom: { type: "node", nodeId: NodeId.make(`node:${name}`) },
archivedAt: name === "archived" ? now : null,
deletedAt: name === "deleted" ? now : null,
});
yield* createRun(child, name === "running" ? "running" : "completed");
if (name === "held" || name === "queued") {
yield* createRun(child, "queued", {
ordinal: 2,
startedAt: null,
...(name === "held" ? { queueHeld: true } : {}),
});
}
children.push(child);
}
const terminalChildren = new Set([children[0]!, children[1]!, children[4]!]);
assert.deepEqual(
new Set(yield* projections.getRecoveryThreadIds("subagent-results")),
terminalChildren,
);
const completed = children[0]!;
const transferId = ContextTransferId.make("transfer:recovery:subagent-result");
yield* projections.apply({
id: EventId.make("event:recovery:subagent-result"),
type: "context-transfer.created",
threadId: parent,
occurredAt: now,
payload: {
id: transferId,
type: "subagent_result",
sourceThreadId: completed,
targetThreadId: parent,
sourcePoint: { threadId: completed },
basePoint: null,
sourceProviderInstanceId: providerInstanceId,
targetProviderInstanceId: providerInstanceId,
targetRunId: null,
status: "pending",
resolution: null,
createdBy: "system",
error: null,
createdAt: now,
updatedAt: now,
consumedAt: null,
},
});
const archivedChild = children[1];
assert.isDefined(archivedChild);
assert.deepEqual(
new Set(yield* projections.getRecoveryThreadIds("subagent-results")),
new Set([archivedChild, children[4]!]),
);
assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
yield* sql`
UPDATE orchestration_v2_projection_context_transfers SET payload_json = '{}'
WHERE context_transfer_id = ${transferId}
`;
assert.deepEqual(
new Set(yield* projections.getUnreadableThreadIds()),
new Set([parent, completed]),
);
}).pipe(Effect.provide(layerTest)),
);
it.effect("recovers only recorded follow-up results with SQLite projections", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const parent = yield* createThread("followup-parent");
const child = yield* createThread("followup-child", {
lineage: { parentThreadId: parent, relationshipToParent: "subagent", rootThreadId: parent },
forkedFrom: { type: "node", nodeId: NodeId.make("node:original") },
});
const originalRunId = yield* createRun(child, "completed");
yield* projections.apply({
id: EventId.make("event:original-result"),
type: "context-transfer.created",
threadId: parent,
occurredAt: now,
payload: {
id: ContextTransferId.make("transfer:original-result"),
type: "subagent_result",
sourceThreadId: child,
targetThreadId: parent,
sourcePoint: { threadId: child, runId: originalRunId },
basePoint: null,
sourceProviderInstanceId: providerInstanceId,
targetProviderInstanceId: providerInstanceId,
targetRunId: null,
status: "pending",
resolution: null,
createdBy: "system",
error: null,
createdAt: now,
updatedAt: now,
consumedAt: null,
},
});
yield* createRun(child, "completed", { ordinal: 2 });
assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []);
const taskId = NodeId.make("node:followup");
yield* createRun(child, "completed", { ordinal: 3, delegatedTaskId: taskId });
yield* createRun(child, "running", { ordinal: 4 });
const task = {
id: taskId,
threadId: parent,
runId: null,
parentNodeId: NodeId.make("node:parent"),
origin: "app_owned" as const,
createdBy: "agent" as const,
driver,
providerInstanceId,
providerThreadId: null,
childThreadId: child,
nativeTaskRef: null,
prompt: "Check the follow-up.",
title: null,
model: null,
status: "running" as const,
result: null,
startedAt: now,
completedAt: null,
updatedAt: now,
};
yield* projections.apply({
id: EventId.make("event:followup-task"),
type: "subagent.updated",
threadId: parent,
occurredAt: now,
payload: task,
});
assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), [child]);
yield* projections.apply({
id: EventId.make("event:followup-settled"),
type: "subagent.updated",
threadId: parent,
occurredAt: now,
payload: { ...task, status: "completed", result: "Follow-up result.", completedAt: now },
});
assert.deepEqual(yield* projections.getRecoveryThreadIds("subagent-results"), []);
}).pipe(Effect.provide(layerTest)),
);
it.effect("includes shared sessions and provider-owned background rosters in recovery", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const first = yield* createThread("shared-first");
const second = yield* createThread("shared-second", { archivedAt: now });
const providerSessionId = ProviderSessionId.make("session:recovery:shared");
for (const threadId of [first, second]) {
yield* projections.apply({
id: EventId.make(`event:${threadId}:session`),
type: "provider-session.attached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerSessionId,
driver,
providerInstanceId,
status: "ready",
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
},
});
}
const roster = yield* createThread("roster");
yield* projections.apply({
id: EventId.make("event:recovery:roster"),
type: "provider-thread.updated",
threadId: roster,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: ProviderThreadId.make("provider-thread:recovery:roster"),
appThreadId: roster,
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
pendingBackgroundTasks: [
{ taskId: "background", description: "Still running", kind: "command" },
],
},
});
const prepared = yield* createThread("prepared-continuation");
const preparedSessionId = ProviderSessionId.make("session:recovery:prepared");
const preparedProviderThreadId = ProviderThreadId.make("provider-thread:recovery:prepared");
yield* projections.apply({
id: EventId.make("event:recovery:prepared-session"),
type: "provider-session.attached",
threadId: prepared,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: preparedSessionId,
driver,
providerInstanceId,
status: "stopped",
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
},
});
yield* projections.apply({
id: EventId.make("event:recovery:prepared-thread"),
type: "provider-thread.updated",
threadId: prepared,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: preparedProviderThreadId,
appThreadId: prepared,
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId: preparedSessionId,
nativeThreadRef: { driver, nativeId: "native:prepared", strength: "strong" },
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: 1,
lastRunOrdinal: 1,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
pendingBackgroundTasks: [],
},
});
const preparedRunId = yield* createRun(prepared, "starting", {
providerThreadId: preparedProviderThreadId,
restartContinuationOfRunId: RunId.make("run:recovery:source"),
});
assert.deepEqual(
new Set(yield* projections.getRecoveryThreadIds("runtime")),
new Set([first, second, roster, prepared]),
);
assert.deepEqual(
new Set(yield* projections.getRecoveryThreadIds("restart-continuations")),
new Set([first, second, prepared]),
);
assert.deepEqual(
yield* projections.getRecoveryThreadIds("restart-continuations", { include: [] }),
[],
);
assert.deepEqual(
yield* projections.getRecoveryThreadIds("restart-continuations", {
exclude: ["project:recovery"],
}),
[],
);
assert.deepEqual(
new Set(
yield* projections.getRecoveryThreadIds("restart-continuations", {
include: ["project:recovery"],
}),
),
new Set([first, second, prepared]),
);
const preparedState = yield* projections.getRuntimeRecoveryProjection(prepared);
assert.deepEqual(
preparedState.providerSessions.map((session) => session.id),
[preparedSessionId],
);
assert.equal(restartContinuationRun(preparedState, yield* DateTime.now)?.id, preparedRunId);
assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
const sql = yield* SqlClient.SqlClient;
yield* sql`
UPDATE orchestration_v2_projection_provider_sessions SET payload_json = '{}'
WHERE provider_session_id = ${providerSessionId}
`;
assert.deepEqual(
new Set(yield* projections.getUnreadableThreadIds()),
new Set([first, second]),
);
}).pipe(Effect.provide(layerTest)),
);
it.effect("bounds streaming recovery reads by candidate runs and preserves message ordering", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const threadId = yield* createThread("streaming");
const other = yield* createThread("streaming-other");
const empty = yield* createThread("streaming-empty");
const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
const observe: Statement.Transformer = (statement) => {
const compiled = statement.compile();
if (compiled[0].includes("SELECT message.payload_json")) statements.push(compiled);
return Effect.succeed(statement);
};
const expected: MessageId[] = [];
const statuses = [
"queued",
"preparing",
"starting",
"running",
"waiting",
"completed",
"interrupted",
"rolled_back",
"failed",
"cancelled",
] as const;
for (const [index, status] of statuses.entries()) {
const runId = yield* createRun(threadId, status, { ordinal: index + 1 });
for (const suffix of ["b", "a", "settled", "wrong-thread"]) {
const id = MessageId.make(`message:streaming:${index}:${suffix}`);
const streaming = suffix !== "settled";
const messageThread = suffix === "wrong-thread" ? other : threadId;
const createdAt = DateTime.add(now, { seconds: -index });
yield* projections.apply({
id: EventId.make(`event:${id}`),
type: "message.updated",
threadId: messageThread,
runId,
occurredAt: now,
payload: {
id,
threadId: messageThread,
runId,
nodeId: null,
role: "assistant",
text: id,
attachments: [],
streaming,
createdAt,
updatedAt: now,
createdBy: "agent",
creationSource: "provider",
},
});
}
if (index < 5)
expected.unshift(
MessageId.make(`message:streaming:${index}:a`),
MessageId.make(`message:streaming:${index}:b`),
);
}
const foreignRun = yield* createRun(other, "running");
yield* sql`
INSERT INTO orchestration_v2_projection_messages
(message_id, thread_id, run_id, role, streaming, created_at, updated_at, payload_json)
VALUES ('foreign-run', ${threadId}, ${foreignRun}, 'assistant', 1, '', '', '{}')
`;
yield* sql`
INSERT INTO orchestration_v2_projection_messages
(message_id, thread_id, run_id, role, streaming, created_at, updated_at, payload_json)
VALUES ('old-history', ${empty}, NULL, 'assistant', 1, '', '', '{}')
`;
const projection = yield* projections
.getRuntimeRecoveryProjection(threadId)
.pipe(Effect.provideService(Statement.CurrentTransformer, observe));
assert.deepEqual(
projection.messages.map((message) => message.id),
expected,
);
const emptyProjection = yield* projections
.getRuntimeRecoveryProjection(empty)
.pipe(Effect.provideService(Statement.CurrentTransformer, observe));
assert.deepEqual(emptyProjection.messages, []);
assert.equal(statements.length, 2);
for (const [query, params] of statements) {
const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
assert.isTrue(
plan.some((row) => row.detail.includes("SEARCH message_run ")),
JSON.stringify(plan),
);
assert.isTrue(
plan.some((row) => row.detail.includes("messages_run_idx")),
JSON.stringify(plan),
);
assert.isFalse(
plan.some((row) => /\bSCAN message\b/.test(row.detail)),
JSON.stringify(plan),
);
assert.isFalse(
plan.some((row) => row.detail.includes("messages_thread_created_idx")),
JSON.stringify(plan),
);
}
}).pipe(Effect.provide(layerTest)),
);
it.effect("bounds runtime provider-thread reads by ownership and preserves recovery liveness", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const threadId = yield* createThread("ownership");
const other = yield* createThread("other-owner");
const ownerNodeId = NodeId.make("node:recovery:owner");
yield* projections.apply({
id: EventId.make("event:recovery:owner"),
type: "node.updated",
threadId,
nodeId: ownerNodeId,
driver,
occurredAt: now,
payload: {
id: ownerNodeId,
threadId,
runId: null,
parentNodeId: null,
rootNodeId: ownerNodeId,
kind: "root_turn",
status: "completed",
countsForRun: false,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
runtimeRequestId: null,
checkpointScopeId: null,
startedAt: now,
completedAt: now,
},
});
const cases = [
{ name: "direct", ownership: "direct", status: "active", ordinal: 2, included: true },
{ name: "owner", ownership: "owner", status: "active", ordinal: null, included: true },
{ name: "owner-only", ownership: "owner", status: "active", ordinal: 1, included: true },
{ name: "subagent", ownership: "subagent", status: "active", ordinal: 1, included: true },
{ name: "duplicate", ownership: "all", status: "active", ordinal: 1, included: true },
{ name: "roster", ownership: "direct", status: "idle", ordinal: 3, included: true },
{ name: "run", ownership: "owner", status: "idle", ordinal: 4, included: true },
{ name: "item", ownership: "subagent", status: "idle", ordinal: 5, included: true },
{ name: "waiting-run", ownership: "direct", status: "idle", ordinal: 6, included: true },
{ name: "settled", ownership: "all", status: "idle", ordinal: 7, included: false },
{ name: "unrelated", ownership: "none", status: "active", ordinal: 8, included: false },
{ name: "unrelated-run", ownership: "none", status: "idle", ordinal: 9, included: false },
] as const;
for (const entry of cases) {
const id = ProviderThreadId.make(`provider-thread:ownership:${entry.name}`);
yield* projections.apply({
id: EventId.make(`event:${id}`),
type: "provider-thread.updated",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id,
appThreadId:
entry.name === "owner-only"
? null
: entry.ownership === "direct" || entry.ownership === "all"
? threadId
: other,
ownerNodeId:
entry.ownership === "owner" || entry.ownership === "all" ? ownerNodeId : null,
driver,
providerInstanceId,
providerSessionId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: entry.status,
firstRunOrdinal: entry.ordinal,
lastRunOrdinal: entry.ordinal,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
pendingBackgroundTasks:
entry.name === "roster"
? [{ taskId: "shell", description: "background", kind: "command" }]
: [],
},
});
if (entry.ownership === "subagent" || entry.ownership === "all") {
for (const suffix of ["a", "b"]) {
yield* sql`
INSERT INTO orchestration_v2_projection_subagents
(subagent_id, thread_id, parent_node_id, provider, provider_thread_id,
origin, status, updated_at, payload_json)
VALUES (${`${id}:${suffix}`}, ${threadId}, ${ownerNodeId}, 'codex', ${id},
'provider_native', 'completed', '', '{}')
`;
}
}
if (entry.name === "run" || entry.name === "unrelated-run") {
yield* createRun(threadId, "waiting", { ordinal: entry.ordinal, providerThreadId: id });
}
}
const waitingProviderThreadId = ProviderThreadId.make("provider-thread:ownership:waiting-run");
const waitingRunId = yield* createRun(threadId, "completed", {
ordinal: 6,
providerThreadId: waitingProviderThreadId,
});
yield* sql`
INSERT INTO orchestration_v2_projection_turn_items
(turn_item_id, thread_id, run_id, node_id, provider_thread_id, ordinal, type,
status, updated_at, payload_json)
VALUES ('item:ownership', ${threadId}, ${waitingRunId}, ${ownerNodeId},
${ProviderThreadId.make("provider-thread:ownership:item")}, 1, 'assistant_message', 'waiting', '', '{}')
`;
const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
const observe: Statement.Transformer = (statement) => {
const compiled = statement.compile();
if (compiled[0].includes("SELECT provider_thread.payload_json")) statements.push(compiled);
return Effect.succeed(statement);
};
const projection = yield* projections
.getRuntimeRecoveryProjection(threadId)
.pipe(Effect.provideService(Statement.CurrentTransformer, observe));
assert.deepEqual(
projection.providerThreads.map((row) => row.id),
cases
.filter((entry) => entry.included)
.toSorted((a, b) => (a.ordinal ?? 0) - (b.ordinal ?? 0) || a.name.localeCompare(b.name))
.map((entry) => ProviderThreadId.make(`provider-thread:ownership:${entry.name}`)),
);
assert.equal(statements.length, 1);
const [query, params] = statements[0]!;
const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
assert.isFalse(
plan.some((row) => /^SCAN provider_thread(?:$| )/.test(row.detail)),
JSON.stringify(plan),
);
assert.isTrue(
plan.some((row) => row.detail.includes("provider_threads_owner_idx")),
JSON.stringify(plan),
);
assert.isFalse(
plan.some((row) => /SEARCH (owner|n) USING INDEX .*nodes_thread/.test(row.detail)),
JSON.stringify(plan),
);
}).pipe(Effect.provide(layerTest)),
);
it.effect("reads the run that owns a background roster, not a queued or resumed one", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const withRoster = Effect.fn(function* (threadId: ThreadId) {
yield* projections.apply({
id: EventId.make(`event:${threadId}:roster`),
type: "provider-thread.updated",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: ProviderThreadId.make(`provider-thread:${threadId}`),
appThreadId: threadId,
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
pendingBackgroundTasks: [
{ taskId: "background", description: "Still running", kind: "command" },
],
},
});
});
const runIds = (threadId: ThreadId) =>
projections
.getRuntimeRecoveryProjection(threadId)
.pipe(Effect.map((state) => state.runs.map((run) => run.id)));
const settled = yield* createThread("roster-before-queue");
const settledRun = yield* createRun(settled, "completed", {
providerThreadId: ProviderThreadId.make(`provider-thread:${settled}`),
});
const queuedRun = yield* createRun(settled, "queued", { ordinal: 2, completedAt: null });
yield* withRoster(settled);
assert.deepEqual(yield* runIds(settled), [settledRun, queuedRun]);
const resumed = yield* createThread("roster-after-resume");
const resumedRun = yield* createRun(resumed, "completed", {
providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
completedAt: DateTime.makeUnsafe("2026-10-03T10:05:00.000Z"),
});
const continuationRun = yield* createRun(resumed, "completed", {
providerThreadId: ProviderThreadId.make(`provider-thread:${resumed}`),
ordinal: 2,
completedAt: DateTime.makeUnsafe("2026-10-03T10:00:00.000Z"),
});
yield* withRoster(resumed);
assert.deepEqual(yield* runIds(resumed), [resumedRun, continuationRun]);
const quiet = yield* createThread("no-roster-before-queue");
yield* createRun(quiet, "completed");
const quietQueued = yield* createRun(quiet, "queued", { ordinal: 2, completedAt: null });
assert.deepEqual(yield* runIds(quiet), [quietQueued]);
}).pipe(Effect.provide(layerTest)),
);
it.effect("wakes the run whose subagent lost its shell, past a wake cut before it started", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const threadId = yield* createThread("subagent-shell");
const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
yield* projections.apply({
id: EventId.make(`event:${threadId}:session`),
type: "provider-session.attached",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerSessionId,
driver,
providerInstanceId,
status: "ready",
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
},
});
const settledRun = yield* createRun(threadId, "completed", { providerThreadId });
yield* createRun(threadId, "starting", {
ordinal: 2,
providerThreadId,
startedAt: null,
completedAt: null,
});
yield* projections.apply({
id: EventId.make(`event:${threadId}:roster`),
type: "provider-thread.updated",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerThreadId,
appThreadId: threadId,
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId,
nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" },
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: 1,
lastRunOrdinal: 2,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
pendingBackgroundTasks: [
{ taskId: "shell", description: "sleep 150", kind: "command", parentTaskId: "agent" },
],
},
});
yield* recovery.reconcile("startup");
const effects = yield* sql<{ readonly effect_id: string }>`
SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
`;
assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
const projection = yield* projections.getThreadProjection(threadId);
assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
{ kind: "subagent", label: "sleep 150 (subagent agent)", id: "agent" },
]);
}).pipe(Effect.provide(layerRecoveryTest)),
);
it.effect("wakes a settled run whose native subagent ran on its own provider thread", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const threadId = yield* createThread("native-subagent");
const parentProviderThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
const childProviderThreadId = ProviderThreadId.make(`provider-thread:${threadId}:child`);
const providerSessionId = ProviderSessionId.make(`session:${threadId}`);
const subagentId = NodeId.make(`node:${threadId}:subagent`);
// An older thread on the same Codex session is recovered first and stops it.
const sibling = yield* createThread("a-shared-session");
yield* createRun(sibling, "completed");
for (const boundThreadId of [sibling, threadId]) {
yield* projections.apply({
id: EventId.make(`event:${boundThreadId}:session`),
type: "provider-session.attached",
threadId: boundThreadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id: providerSessionId,
driver,
providerInstanceId,
status: "ready",
cwd: "/workspace",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
},
});
}
const settledRun = yield* createRun(threadId, "completed", {
providerThreadId: parentProviderThreadId,
});
for (const [id, nativeId] of [
[parentProviderThreadId, "native-thread"],
[childProviderThreadId, "agent-thread"],
] as const) {
yield* projections.apply({
id: EventId.make(`event:${id}`),
type: "provider-thread.updated",
threadId,
driver,
providerInstanceId,
occurredAt: now,
payload: {
id,
appThreadId:
id === parentProviderThreadId ? threadId : ThreadId.make(`${threadId}:child`),
ownerNodeId: null,
driver,
providerInstanceId,
providerSessionId,
nativeThreadRef: { driver, nativeId, strength: "strong" },
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: id === parentProviderThreadId ? 1 : null,
lastRunOrdinal: id === parentProviderThreadId ? 1 : null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
},
});
}
const subagent = {
threadId,
runId: settledRun,
origin: "provider_native",
driver,
providerInstanceId,
childThreadId: null,
prompt: "audit src",
status: "running",
result: null,
startedAt: now,
completedAt: null,
updatedAt: now,
} as const;
yield* projections.apply({
id: EventId.make(`event:${threadId}:subagent`),
type: "subagent.updated",
threadId,
occurredAt: now,
payload: {
...subagent,
id: subagentId,
parentNodeId: NodeId.make(`node:${threadId}:root`),
createdBy: "agent",
providerThreadId: childProviderThreadId,
nativeTaskRef: { driver, nativeId: "call_1:agent-thread", strength: "strong" },
title: null,
model: null,
},
});
yield* projections.apply({
id: EventId.make(`event:${threadId}:subagent-item`),
type: "turn-item.updated",
threadId,
occurredAt: now,
payload: {
...subagent,
id: TurnItemId.make(`item:${threadId}:subagent`),
nodeId: subagentId,
providerThreadId: childProviderThreadId,
providerTurnId: null,
nativeItemRef: { driver, nativeId: "call_1:agent-thread", strength: "strong" },
parentItemId: null,
ordinal: 1,
type: "subagent",
title: "audit",
subagentId,
},
});
yield* recovery.reconcile("startup");
const effects = yield* sql<{ readonly effect_id: string }>`
SELECT effect_id FROM orchestration_v2_effect_outbox WHERE thread_id = ${threadId}
`;
assert.deepEqual(effects, [{ effect_id: `effect:restart-continuation:${settledRun}` }]);
const projection = yield* projections.getThreadProjection(threadId);
assert.deepEqual(projection.runs[0]?.restartCancelledBackgroundWork, [
{
kind: "subagent",
label: "audit (id agent-thread)",
id: TurnItemId.make(`item:${threadId}:subagent`),
},
]);
}).pipe(Effect.provide(layerRecoveryTest)),
);
it.effect("marks fork descendants unreadable when their source is missing or corrupt", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const source = yield* createThread("source");
const sourceRun = yield* createRun(source, "completed");
const fork = yield* createThread("fork", {
forkedFrom: { type: "run", threadId: source, runId: sourceRun },
});
const descendant = yield* createThread("descendant", {
forkedFrom: { type: "run", threadId: fork, runId: RunId.make("run:fork") },
});
assert.deepEqual(yield* projections.getUnreadableThreadIds(), []);
yield* sql`
UPDATE orchestration_v2_projection_runs SET payload_json = '{}'
WHERE run_id = ${sourceRun}
`;
assert.deepEqual(
new Set(yield* projections.getUnreadableThreadIds()),
new Set([source, fork, descendant]),
);
yield* sql`DELETE FROM orchestration_v2_projection_threads WHERE thread_id = ${source}`;
assert.deepEqual(
new Set(yield* projections.getUnreadableThreadIds()),
new Set([fork, descendant]),
);
}).pipe(Effect.provide(layerTest)),
);
it.effect("startup ends the provider turn of a native child root it cancels", () =>
Effect.gen(function* () {
const projections = yield* ProjectionStore.ProjectionStoreV2;
const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
const now = yield* DateTime.now;
const parentThreadId = yield* createThread("native-child-root-turn-parent");
const threadId = yield* createThread("native-child-root-turn");
const rootNodeId = NodeId.make(`node:${threadId}:thread-root`);
const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
const providerTurnId = ProviderTurnId.make(`provider-turn:${threadId}`);
yield* projections.apply({
id: EventId.make(`event:${threadId}:root`),
type: "node.updated",
threadId,
nodeId: rootNodeId,
driver,
occurredAt: now,
payload: {
id: rootNodeId,
threadId,
runId: null,
parentNodeId: null,
rootNodeId,
kind: "root_turn",
status: "running",
countsForRun: false,
providerThreadId,
providerTurnId,
nativeItemRef: null,
runtimeRequestId: null,
checkpointScopeId: null,
startedAt: now,
completedAt: null,
},
});
// A native subagent's turn belongs to no run attempt.
yield* projections.apply({
id: EventId.make(`event:${threadId}:provider-turn`),
type: "provider-turn.updated",
threadId,
nodeId: rootNodeId,
driver,
occurredAt: now,
payload: {
id: providerTurnId,
providerThreadId,
nodeId: rootNodeId,
runAttemptId: null,
nativeTurnRef: null,
ordinal: 1,
status: "running",
startedAt: now,
completedAt: null,
},
});
const subagentId = NodeId.make(`node:${parentThreadId}:subagent`);
yield* projections.apply({
id: EventId.make(`event:${parentThreadId}:subagent`),
type: "subagent.updated",
threadId: parentThreadId,
occurredAt: now,
payload: {
id: subagentId,
threadId: parentThreadId,
runId: null,
parentNodeId: NodeId.make(`node:${parentThreadId}:root`),
origin: "provider_native",
createdBy: "agent",
driver,
providerInstanceId,
providerThreadId,
childThreadId: threadId,
nativeTaskRef: null,
title: null,
model: null,
prompt: "sleep 300",
status: "running",
result: null,
startedAt: now,
completedAt: null,
updatedAt: now,
},
});
yield* recovery.reconcile("startup");
const projection = yield* projections.getThreadProjection(threadId);
assert.deepEqual(
{
root: projection.nodes.find((node) => node.id === rootNodeId)?.status,
providerTurn: projection.providerTurns.find((turn) => turn.id === providerTurnId)?.status,
},
{ root: "cancelled", providerTurn: "cancelled" },
"a root startup recovery cancelled left its provider turn running",
);
}).pipe(Effect.provide(layerRecoveryTest)),
);