import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it, vi } from "@effect/vitest";
import {
CheckpointId,
CommandId,
EventId,
MessageId,
type ModelSelection,
NodeId,
type OrchestrationV2Run,
ProjectId,
ProviderDriverKind,
ProviderInstanceId,
ProviderThreadId,
RunId,
RuntimeRequestId,
ThreadId,
TurnItemId,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Deferred from "effect/Deferred";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Stream from "effect/Stream";
import * as Tracer from "effect/Tracer";
import * as TestClock from "effect/testing/TestClock";
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
import * as ServerConfig from "../config.ts";
import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts";
import * as ProjectService from "../project/ProjectService.ts";
import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
import * as VcsProcess from "../vcs/VcsProcess.ts";
import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
import * as EventSink from "./EventSink.ts";
import * as Orchestrator from "./Orchestrator.ts";
import { continueRestartedRun } from "./RestartContinuation.ts";
import * as RuntimeLayer from "./runtimeLayer.ts";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProviderContinuationService from "./ProviderContinuationService.ts";
const layerPlatformTest = Layer.merge(
Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({
resolveLink: () => Effect.die("unused title link"),
}),
);
const layerServerConfig = ServerConfig.layerTest(process.cwd(), {
prefix: "t3-orchestration-v2-delegated-completion-",
});
const modelSelection = {
instanceId: ProviderInstanceId.make("codex"),
model: "gpt-5.4",
} satisfies ModelSelection;
const layerVcsDriverRegistryTest = VcsDriverRegistry.layer.pipe(
Layer.provide(VcsProcess.layer),
Layer.provide(layerServerConfig),
Layer.provide(layerPlatformTest),
);
const layerCheckpointStoreTest = CheckpointStore.layer.pipe(
Layer.provide(layerVcsDriverRegistryTest),
);
const driver = ProviderDriverKind.make("codex");
const orchestrationAdapter = {
instanceId: modelSelection.instanceId,
driver,
getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }),
openSession: () => Effect.die("sessions are not used by delegated completion tests"),
} as ProviderAdapter.ProviderAdapterV2["Service"];
const providerInstance = {
instanceId: modelSelection.instanceId,
driverKind: driver,
continuationIdentity: {
driverKind: driver,
continuationKey: "codex:test",
},
displayName: "Codex test",
enabled: true,
snapshot: { getSnapshot: Effect.succeed({}) } as unknown as ProviderInstance["snapshot"],
orchestrationAdapter,
textGeneration: {} as ProviderInstance["textGeneration"],
} satisfies ProviderInstance;
const layerTestProviderInstanceRegistry = Layer.succeed(
ProviderInstanceRegistry.ProviderInstanceRegistry,
{
getInstance: (instanceId) =>
Effect.succeed(instanceId === providerInstance.instanceId ? providerInstance : undefined),
listInstances: Effect.succeed([providerInstance]),
listUnavailable: Effect.succeed([]),
streamChanges: Stream.empty,
subscribeChanges: Effect.never,
},
);
const layerTest = Layer.mergeAll(
RuntimeLayer.layer,
RuntimeLayer.layerEventSink,
ThreadCommandExecutor.layer,
).pipe(
Layer.provideMerge(RuntimeLayer.layerProjectService),
Layer.provide(
Layer.mock(WorkspacePaths.WorkspacePaths)({
normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(workspaceRoot),
}),
),
Layer.provide(ProviderTurnStartServiceTestkit.layer),
Layer.provide(
Layer.succeed(ProjectEnrichmentService.ProjectEnrichmentService, {
peek: () =>
Effect.succeed({
repositoryIdentity: null,
faviconPath: null,
repositoryIdentityResolved: false,
}),
request: () => Effect.void,
readRepositoryIdentity: () => Effect.succeed(null),
getAvailable: () =>
Effect.succeed({
repositoryIdentity: null,
faviconPath: null,
repositoryIdentityResolved: false,
}),
invalidate: () => Effect.void,
subscribeChanges: Effect.never,
}),
),
Layer.provide(McpSessionRegistryTestkit.layer),
Layer.provide(McpProviderSessions.layer),
Layer.provide(SqlitePersistence.layerMemory),
Layer.provide(layerCheckpointStoreTest),
Layer.provide(layerServerConfig),
Layer.provide(ServerSettings.layerTest()),
Layer.provide(layerTestProviderInstanceRegistry),
Layer.provide(layerPlatformTest),
);
// ProviderContinuationService consumes the orchestrator's continuation queue, as in production.
const layerContinuationWorkerTest = ProviderContinuationService.layer.pipe(
Layer.provide(Layer.merge(ProviderContinuationRequests.layer, IdAllocator.layer)),
Layer.provideMerge(layerTest),
);
const seedParentWithTerminalTask = (input: {
readonly threadId: ThreadId;
readonly projectId: ProjectId;
readonly runId: RunId;
readonly rootNodeId: NodeId;
readonly taskId: NodeId;
readonly deliveryState: "pending" | "delivered" | "claimed" | "acknowledged" | "disposed";
readonly completionWake?: "always" | "settled_only";
readonly deliveryTaskIds?: ReadonlyArray<NodeId>;
readonly parentRunStatus?: "running" | "completed";
readonly siblingTasks?: ReadonlyArray<{
readonly id: NodeId;
readonly deliveryState: "delivered" | "claimed";
}>;
readonly now: DateTime.Utc;
}) =>
Effect.gen(function* () {
const projects = yield* ProjectService.ProjectService;
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const providerThreadId = ProviderThreadId.make(
`provider-thread:${String(input.threadId).replace("thread:", "")}`,
);
yield* projects.create({
commandId: CommandId.make(`command:seed-project:${input.threadId}`),
projectId: input.projectId,
title: "Delegated completion delivery",
workspaceRoot: `/workspace/${input.projectId}`,
});
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make(`command:seed-create:${input.threadId}`),
threadId: input.threadId,
projectId: input.projectId,
title: "Delegated completion delivery",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
});
yield* eventSink.write({
commandId: CommandId.make(`command:seed-projection:${input.threadId}`),
events: [
{
id: EventId.make(`event:seed-provider-thread:${input.threadId}`),
type: "provider-thread.updated",
threadId: input.threadId,
driver,
providerInstanceId: modelSelection.instanceId,
occurredAt: input.now,
payload: {
id: providerThreadId,
driver,
providerInstanceId: modelSelection.instanceId,
providerSessionId: null,
appThreadId: input.threadId,
ownerNodeId: input.rootNodeId,
nativeThreadRef: {
driver,
nativeId: `native:${input.threadId}`,
strength: "strong",
},
nativeConversationHeadRef: null,
status: "active",
firstRunOrdinal: 1,
lastRunOrdinal: 1,
handoffIds: [],
forkedFrom: null,
createdAt: input.now,
updatedAt: input.now,
},
},
{
id: EventId.make(`event:seed-run:${input.threadId}`),
type: "run.updated",
threadId: input.threadId,
runId: input.runId,
nodeId: input.rootNodeId,
providerInstanceId: modelSelection.instanceId,
occurredAt: input.now,
payload: {
id: input.runId,
threadId: input.threadId,
ordinal: 1,
providerInstanceId: modelSelection.instanceId,
modelSelection,
providerThreadId,
userMessageId: MessageId.make(`message:seed-user:${input.threadId}`),
rootNodeId: input.rootNodeId,
activeAttemptId: null,
status: input.parentRunStatus ?? "running",
requestedAt: input.now,
startedAt: input.now,
completedAt: input.parentRunStatus === "completed" ? input.now : null,
checkpointId: null,
contextHandoffId: null,
delegatedCompletion: {
disposition: "open",
nextGeneration: 2,
delivery:
input.deliveryTaskIds === undefined
? null
: {
generation: 1,
messageId: MessageId.make(`message:delegated-delivery:${input.threadId}`),
taskIds: input.deliveryTaskIds,
},
},
},
},
...[
{ id: input.taskId, deliveryState: input.deliveryState },
...(input.siblingTasks ?? []),
].map((task) => ({
id: EventId.make(`event:seed-task:${input.threadId}:${task.id}`),
type: "subagent.updated" as const,
threadId: input.threadId,
runId: input.runId,
nodeId: task.id,
driver,
providerInstanceId: modelSelection.instanceId,
occurredAt: input.now,
payload: {
id: task.id,
threadId: input.threadId,
runId: input.runId,
parentNodeId: input.rootNodeId,
origin: "app_owned" as const,
createdBy: "agent" as const,
driver,
providerInstanceId: modelSelection.instanceId,
providerThreadId: null,
childThreadId: null,
nativeTaskRef: null,
prompt: "Inspect the delivered ownership edge.",
title: null,
model: null,
completionWake: input.completionWake ?? "settled_only",
completionDelivery: {
state: task.deliveryState,
observedByRunId: task.deliveryState === "acknowledged" ? input.runId : null,
},
status: "completed" as const,
result: "child finished",
startedAt: input.now,
completedAt: input.now,
updatedAt: input.now,
},
})),
],
});
});
it.layer(layerTest)("delegated completion delivery repairs", (it) => {
it.effect(
"interrupting a completion wake disposes both cohorts without reviving them on late results",
() =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("stop-wake-parent");
const originRunId = RunId.make("stop-wake-origin");
const incomingTaskId = NodeId.make("stop-wake-incoming");
yield* seedParentWithTerminalTask({
threadId,
projectId: ProjectId.make("stop-wake-project"),
runId: originRunId,
rootNodeId: NodeId.make("stop-wake-origin-root"),
taskId: incomingTaskId,
deliveryState: "claimed",
deliveryTaskIds: [incomingTaskId],
completionWake: "always",
now,
});
const origin = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
yield* sink.write({
events: [
{
...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
payload: { ...origin, status: "completed", completedAt: now },
},
],
});
const delivery = origin.delegatedCompletion!.delivery!;
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "agent",
creationSource: "server",
commandId: CommandId.make("stop-wake-start"),
threadId,
messageId: delivery.messageId,
text: "Delegated task completed",
attachments: [],
modelSelection,
dispatchMode: { type: "queue_after_active" },
delegatedCompletion: {
parentRunId: originRunId,
generation: delivery.generation,
taskIds: delivery.taskIds,
},
});
yield* orchestrator.resumeQueuedRuns;
const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
(run) => run.userMessageId === delivery.messageId,
)!;
yield* orchestrator.dispatch({
type: "delegated_task.request",
createdBy: "agent",
creationSource: "mcp",
commandId: CommandId.make("stop-wake-delegate"),
parentThreadId: threadId,
parentRunId: wake.id,
parentNodeId: wake.rootNodeId!,
task: "Do the next round",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
completionWake: "always",
});
const delegated = (yield* orchestrator.getThreadProjection(threadId)).subagents.find(
(task) => task.runId === wake.id,
)!;
yield* orchestrator.dispatch({
type: "run.interrupt",
commandId: CommandId.make("stop-wake-interrupt"),
threadId,
runId: wake.id,
});
const stopped = yield* orchestrator.getThreadProjection(threadId);
for (const runId of [originRunId, wake.id]) {
const run = stopped.runs.find((row) => row.id === runId)!;
assert.equal(run.delegatedCompletion?.disposition, "stopped");
assert.isNull(run.delegatedCompletion?.delivery);
}
assert.equal(stopped.runs.find((run) => run.id === wake.id)?.status, "interrupted");
for (const taskId of [incomingTaskId, delegated.id]) {
assert.equal(
stopped.subagents.find((task) => task.id === taskId)?.completionDelivery?.state,
"disposed",
);
}
assert.equal(
stopped.subagents.find((task) => task.id === incomingTaskId)?.result,
"child finished",
);
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make("stop-wake-newer-work"),
threadId,
messageId: MessageId.make("stop-wake-newer-work"),
text: "Start unrelated work",
attachments: [],
modelSelection,
dispatchMode: { type: "queue_after_active" },
});
yield* orchestrator.resumeQueuedRuns;
const newer = yield* orchestrator.getThreadProjection(threadId);
assert.equal(newer.runs.at(-1)?.status, "starting");
const childThreadId = delegated.childThreadId!;
const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!;
const afterSequence = yield* sink.latestSequence();
yield* sink.write({
events: [
{
...runEvent({
threadId: childThreadId,
runId: childRun.id,
ordinal: childRun.ordinal,
status: "completed",
now,
}),
payload: { ...childRun, status: "completed", completedAt: now },
},
],
});
yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
Stream.filter(
(stored) =>
stored.event.type === "subagent.updated" &&
stored.event.payload.id === delegated.id &&
stored.event.payload.status === "completed",
),
Stream.take(1),
Stream.runDrain,
);
const finished = yield* orchestrator.getThreadProjection(threadId);
const lateTask = finished.subagents.find((task) => task.id === delegated.id)!;
assert.equal(lateTask.status, "completed");
assert.isNotNull(lateTask.result);
assert.equal(lateTask.completionDelivery?.state, "disposed");
assert.deepEqual(finished.runs, newer.runs);
assert.deepEqual(finished.messages, newer.messages);
}),
);
it.effect("acceptance batches pending siblings without acknowledging their results", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("mailbox-batch");
const runId = RunId.make("mailbox-parent");
const taskId = NodeId.make("mailbox-first");
const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
yield* seedParentWithTerminalTask({
threadId,
runId,
projectId: ProjectId.make("mailbox-project"),
rootNodeId: NodeId.make("mailbox-root"),
taskId,
deliveryState: "claimed",
completionWake: "always",
deliveryTaskIds: [taskId],
now,
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const task = projection.subagents[0]!;
const pendingIds = [NodeId.make("mailbox-second"), NodeId.make("mailbox-third")];
yield* sink.write({
events: [
{
id: EventId.make("mailbox-message"),
type: "message.updated",
threadId,
runId,
occurredAt: now,
payload: {
id: messageId,
threadId,
runId,
nodeId: task.parentNodeId,
role: "user",
text: "Background task finished",
attachments: [],
streaming: false,
createdBy: "agent",
creationSource: "server",
createdAt: now,
updatedAt: now,
delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
},
},
...pendingIds.map((id) => ({
id: EventId.make(`event:${id}`),
type: "subagent.updated" as const,
threadId,
runId,
nodeId: id,
occurredAt: now,
payload: {
...task,
id,
completionDelivery: { state: "pending" as const, observedByRunId: null },
},
})),
],
});
yield* orchestrator.dispatch({
type: "notification.delivery.accept",
commandId: CommandId.make("accept-first"),
threadId,
messageId,
});
const accepted = yield* orchestrator.getThreadProjection(threadId);
assert.equal(
accepted.subagents.find((row) => row.id === taskId)?.completionDelivery?.state,
"delivered",
);
const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
assert.deepEqual(cohort?.delivery?.taskIds, pendingIds);
assert.equal(cohort?.delivery?.generation, 2);
for (const id of pendingIds) {
assert.deepEqual(accepted.subagents.find((row) => row.id === id)?.completionDelivery, {
state: "claimed",
observedByRunId: null,
});
}
yield* orchestrator.dispatch({
type: "notification.delivery.accept",
commandId: CommandId.make("repeat-old-acceptance"),
threadId,
messageId,
});
const duplicate = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(duplicate.runs.find((row) => row.id === runId)?.delegatedCompletion, cohort);
}),
);
it.effect("acceptance batches a settled_only sibling once its spawning run ended", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("mailbox-mixed");
const runId = RunId.make("mailbox-mixed-parent");
const taskId = NodeId.make("mailbox-mixed-always");
const siblingId = NodeId.make("mailbox-mixed-settled-only");
const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
yield* seedParentWithTerminalTask({
threadId,
runId,
projectId: ProjectId.make("mailbox-mixed-project"),
rootNodeId: NodeId.make("mailbox-mixed-root"),
taskId,
deliveryState: "claimed",
completionWake: "always",
deliveryTaskIds: [taskId],
now,
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const task = projection.subagents[0]!;
const spawningRun = projection.runs.find((row) => row.id === runId)!;
yield* sink.write({
events: [
{
...runEvent({ threadId, runId, ordinal: 1, status: "cancelled", now }),
payload: { ...spawningRun, status: "cancelled" as const, completedAt: now },
},
runEvent({
threadId,
runId: RunId.make("mailbox-mixed-continuation"),
ordinal: 2,
status: "running",
now,
}),
{
id: EventId.make("mailbox-mixed-message"),
type: "message.updated",
threadId,
runId,
occurredAt: now,
payload: {
id: messageId,
threadId,
runId,
nodeId: task.parentNodeId,
role: "user",
text: "Background task finished",
attachments: [],
streaming: false,
createdBy: "agent",
creationSource: "server",
createdAt: now,
updatedAt: now,
delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
},
},
{
id: EventId.make(`event:${siblingId}`),
type: "subagent.updated" as const,
threadId,
runId,
nodeId: siblingId,
occurredAt: now,
payload: {
...task,
id: siblingId,
completionWake: "settled_only" as const,
completionDelivery: { state: "pending" as const, observedByRunId: null },
},
},
],
});
yield* orchestrator.dispatch({
type: "notification.delivery.accept",
commandId: CommandId.make("accept-mixed"),
threadId,
messageId,
});
const accepted = yield* orchestrator.getThreadProjection(threadId);
const cohort = accepted.runs.find((row) => row.id === runId)?.delegatedCompletion;
assert.deepEqual(cohort?.delivery?.taskIds, [siblingId]);
assert.equal(
accepted.subagents.find((row) => row.id === siblingId)?.completionDelivery?.state,
"claimed",
);
}),
);
it.effect("builds completion text and metadata from the same live cohort", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:delegated-delivery-live-cohort");
const projectId = ProjectId.make("project:delegated-delivery-live-cohort");
const runId = RunId.make("run:delegated-delivery-live-cohort");
const rootNodeId = NodeId.make("node:delegated-delivery-live-cohort-root");
const firstTaskId = NodeId.make("node:delegated-delivery-live-cohort-first");
const secondTaskId = NodeId.make("node:delegated-delivery-live-cohort-second");
const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
yield* seedParentWithTerminalTask({
threadId,
projectId,
runId,
rootNodeId,
taskId: firstTaskId,
deliveryState: "claimed",
completionWake: "always",
deliveryTaskIds: [firstTaskId, secondTaskId],
now,
});
yield* orchestrator.dispatch({
type: "message.dispatch",
commandId: CommandId.make("command:delegated-delivery-live-cohort"),
threadId,
messageId,
text: `Delegated task ${firstTaskId} reached a terminal state.`,
attachments: [],
dispatchMode: { type: "queue_after_active" },
createdBy: "agent",
creationSource: "server",
delegatedCompletion: {
parentRunId: runId,
generation: 1,
taskIds: [firstTaskId],
},
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const message = projection.messages.find((candidate) => candidate.id === messageId);
assert.deepEqual(message?.delegatedCompletion?.taskIds, [firstTaskId, secondTaskId]);
assert.include(message?.text ?? "", String(firstTaskId));
assert.include(message?.text ?? "", String(secondTaskId));
assert.include(message?.text ?? "", "task_status");
assert.include(message?.text ?? "", "child finished");
}),
);
it.effect("counts already delivered siblings in the completion notice", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:delegated-delivery-notice-count");
const runId = RunId.make("run:delegated-delivery-notice-count");
const firstTaskId = NodeId.make("node:delegated-delivery-notice-count-first");
const secondTaskId = NodeId.make("node:delegated-delivery-notice-count-second");
yield* seedParentWithTerminalTask({
threadId,
projectId: ProjectId.make("project:delegated-delivery-notice-count"),
runId,
rootNodeId: NodeId.make("node:delegated-delivery-notice-count-root"),
taskId: firstTaskId,
deliveryState: "claimed",
completionWake: "always",
deliveryTaskIds: [firstTaskId, secondTaskId],
parentRunStatus: "completed",
siblingTasks: [
{ id: secondTaskId, deliveryState: "claimed" },
{
id: NodeId.make("node:delegated-delivery-notice-count-earlier"),
deliveryState: "delivered",
},
],
now,
});
yield* orchestrator.dispatch({
type: "message.dispatch",
commandId: CommandId.make("command:delegated-delivery-notice-count"),
threadId,
messageId: MessageId.make(`message:delegated-delivery:${threadId}`),
text: "Delegated tasks reached a terminal state.",
attachments: [],
dispatchMode: { type: "queue_after_active" },
createdBy: "agent",
creationSource: "server",
delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [firstTaskId] },
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const notice = projection.turnItems.find((item) => item.type === "notification");
assert.match(
notice?.type === "notification" ? notice.summary : "",
/^2 of 3 delegated tasks finished:/,
);
}),
);
it.effect.each([
{
name: "oversized UTF-8 result",
first: "界".repeat(6_000),
second: "small result",
inlineFirst: false,
inlineSecond: true,
},
{
name: "combined results",
first: "a".repeat(10_000),
second: "b".repeat(10_000),
inlineFirst: true,
inlineSecond: false,
},
])("uses task_status for $name beyond the inline budget", (testCase) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make(`thread:inline-budget:${testCase.name}`);
const runId = RunId.make(`run:inline-budget:${testCase.name}`);
const firstTaskId = NodeId.make(`node:inline-first:${testCase.name}`);
const secondTaskId = NodeId.make(`node:inline-second:${testCase.name}`);
const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
yield* seedParentWithTerminalTask({
threadId,
projectId: ProjectId.make(`project:inline-budget:${testCase.name}`),
runId,
rootNodeId: NodeId.make(`node:inline-budget:${testCase.name}`),
taskId: firstTaskId,
deliveryState: "claimed",
completionWake: "always",
deliveryTaskIds: [firstTaskId, secondTaskId],
now,
});
const parent = yield* orchestrator.getThreadProjection(threadId);
const task = parent.subagents[0]!;
yield* sink.write({
events: [
{ id: firstTaskId, result: testCase.first, status: "failed" as const },
{ id: secondTaskId, result: testCase.second, status: "cancelled" as const },
].map(({ id, result, status }) => ({
id: EventId.make(`event:inline-budget:${id}`),
type: "subagent.updated" as const,
threadId,
runId,
nodeId: id,
occurredAt: now,
payload: { ...task, id, result, status, title: "Inspect regression" },
})),
});
yield* orchestrator.dispatch({
type: "message.dispatch",
commandId: CommandId.make(`command:inline-budget:${testCase.name}`),
threadId,
messageId,
text: "Delegated tasks completed.",
attachments: [],
dispatchMode: { type: "queue_after_active" },
createdBy: "agent",
creationSource: "server",
delegatedCompletion: {
parentRunId: runId,
generation: 1,
taskIds: [firstTaskId, secondTaskId],
},
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const message = projection.messages.find((candidate) => candidate.id === messageId)!;
assert.equal(message.text.includes(testCase.first), testCase.inlineFirst);
assert.equal(message.text.includes(testCase.second), testCase.inlineSecond);
const deferredTaskId = testCase.inlineFirst ? secondTaskId : firstTaskId;
assert.include(
message.text,
`Delegated task ${deferredTaskId} (Inspect regression) [${testCase.inlineFirst ? "cancelled" : "failed"}]`,
);
assert.include(message.text, `Use task_status with taskId ${deferredTaskId}`);
assert.deepEqual(message.delegatedCompletion?.taskIds, [firstTaskId, secondTaskId]);
assert.equal(
projection.subagents.find((candidate) => candidate.id === deferredTaskId)?.result,
testCase.inlineFirst ? testCase.second : testCase.first,
);
}),
);
it.effect("does not re-offer when wake-policy upgrades after delivered ownership settled", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:delegated-delivery-a1");
const projectId = ProjectId.make("project:delegated-delivery-a1");
const runId = RunId.make("run:delegated-delivery-a1");
const rootNodeId = NodeId.make("node:delegated-delivery-a1-root");
const taskId = NodeId.make("node:delegated-delivery-a1-task");
yield* seedParentWithTerminalTask({
threadId,
projectId,
runId,
rootNodeId,
taskId,
deliveryState: "delivered",
completionWake: "settled_only",
now,
});
const upgrade = yield* orchestrator.dispatch({
type: "delegated_task.wake-policy",
commandId: CommandId.make("command:delegated-delivery-a1:wake-policy"),
parentThreadId: threadId,
taskId,
completionWake: "always",
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const task = projection.subagents.find((candidate) => candidate.id === taskId);
const parentRun = projection.runs.find((candidate) => candidate.id === runId);
assert.equal(task?.completionWake, "always");
assert.deepEqual(task?.completionDelivery, {
state: "delivered",
observedByRunId: null,
});
assert.deepEqual(parentRun?.delegatedCompletion, {
disposition: "open",
nextGeneration: 2,
delivery: null,
});
assert.isFalse(
upgrade.storedEvents.some(
(stored) =>
stored.event.type === "subagent.updated" &&
stored.event.payload.id === taskId &&
stored.event.payload.completionDelivery?.state === "claimed",
),
);
assert.isFalse(
upgrade.storedEvents.some(
(stored) =>
stored.event.type === "run.updated" &&
stored.event.payload.id === runId &&
stored.event.payload.delegatedCompletion?.delivery !== null &&
stored.event.payload.delegatedCompletion?.delivery !== undefined,
),
);
}),
);
it.effect(
"treats repeated acknowledge and dispose with distinct command IDs as successful no-ops",
() =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:delegated-delivery-a2");
const projectId = ProjectId.make("project:delegated-delivery-a2");
const runId = RunId.make("run:delegated-delivery-a2");
const rootNodeId = NodeId.make("node:delegated-delivery-a2-root");
const taskId = NodeId.make("node:delegated-delivery-a2-task");
yield* seedParentWithTerminalTask({
threadId,
projectId,
runId,
rootNodeId,
taskId,
deliveryState: "delivered",
completionWake: "always",
now,
});
const firstAck = yield* orchestrator.dispatch({
type: "delegated_task.completion-delivery.acknowledge",
commandId: CommandId.make("command:delegated-delivery-a2:ack-task-status"),
parentThreadId: threadId,
taskId,
observedByRunId: runId,
});
const secondAck = yield* orchestrator.dispatch({
type: "delegated_task.completion-delivery.acknowledge",
commandId: CommandId.make("command:delegated-delivery-a2:ack-thread-read"),
parentThreadId: threadId,
taskId,
observedByRunId: runId,
});
const firstAckTask = firstAck.storedEvents.find(
(stored) =>
stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
);
const secondAckTask = secondAck.storedEvents.find(
(stored) =>
stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
);
assert.isDefined(firstAckTask);
assert.isDefined(secondAckTask);
if (
firstAckTask?.event.type !== "subagent.updated" ||
secondAckTask?.event.type !== "subagent.updated"
) {
return yield* Effect.die(new Error("Acknowledge events missing."));
}
assert.equal(firstAckTask.event.payload.completionDelivery?.state, "acknowledged");
assert.equal(secondAckTask.event.payload.completionDelivery?.state, "acknowledged");
assert.deepEqual(
secondAckTask.event.payload.completionDelivery,
firstAckTask.event.payload.completionDelivery,
);
assert.deepEqual(
secondAckTask.event.payload.updatedAt,
firstAckTask.event.payload.updatedAt,
);
assert.equal(secondAck.storedEvents.length, 1);
const afterAck = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(
afterAck.subagents.find((candidate) => candidate.id === taskId)?.completionDelivery,
{
state: "acknowledged",
observedByRunId: runId,
},
);
const firstDispose = yield* orchestrator.dispatch({
type: "delegated_task.completion-delivery.dispose",
commandId: CommandId.make("command:delegated-delivery-a2:dispose-task-status"),
parentThreadId: threadId,
taskId,
});
const secondDispose = yield* orchestrator.dispatch({
type: "delegated_task.completion-delivery.dispose",
commandId: CommandId.make("command:delegated-delivery-a2:dispose-thread-read"),
parentThreadId: threadId,
taskId,
});
const firstDisposeTask = firstDispose.storedEvents.find(
(stored) =>
stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
);
const secondDisposeTask = secondDispose.storedEvents.find(
(stored) =>
stored.event.type === "subagent.updated" && stored.event.payload.id === taskId,
);
assert.isDefined(firstDisposeTask);
assert.isDefined(secondDisposeTask);
if (
firstDisposeTask?.event.type !== "subagent.updated" ||
secondDisposeTask?.event.type !== "subagent.updated"
) {
return yield* Effect.die(new Error("Dispose events missing."));
}
assert.equal(firstDisposeTask.event.payload.completionDelivery?.state, "disposed");
assert.equal(secondDisposeTask.event.payload.completionDelivery?.state, "disposed");
assert.deepEqual(
secondDisposeTask.event.payload.completionDelivery,
firstDisposeTask.event.payload.completionDelivery,
);
assert.equal(secondDispose.storedEvents.length, 1);
const afterDispose = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(
afterDispose.subagents.find((candidate) => candidate.id === taskId)?.completionDelivery,
{
state: "disposed",
observedByRunId: null,
},
);
const acknowledgeAfterDispose = yield* orchestrator.dispatch({
type: "delegated_task.completion-delivery.acknowledge",
commandId: CommandId.make("command:delegated-delivery-a2:ack-after-dispose"),
parentThreadId: threadId,
taskId,
observedByRunId: runId,
});
const acknowledgedTask = acknowledgeAfterDispose.storedEvents.find(
(stored) => stored.event.type === "subagent.updated",
);
if (acknowledgedTask?.event.type !== "subagent.updated") {
return yield* Effect.die(new Error("Acknowledge-after-dispose event missing."));
}
assert.deepEqual(acknowledgedTask.event.payload.completionDelivery, {
state: "disposed",
observedByRunId: null,
});
assert.equal(acknowledgeAfterDispose.storedEvents.length, 1);
const afterStaleAcknowledge = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(
afterStaleAcknowledge.subagents.find((candidate) => candidate.id === taskId)
?.completionDelivery,
{
state: "disposed",
observedByRunId: null,
},
);
}),
);
it.effect.each([
{ command: "acknowledge", before: "running", after: "failed", capture: false },
{ command: "acknowledge", before: "waiting", after: "completed", capture: true },
{ command: "accept", before: "waiting", after: "completed", capture: true },
{ command: "wake-policy", before: "running", after: "completed", capture: true },
] as const)(
"keeps a $after run that settled between $command's read and commit",
({ command, before, after, capture }) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const race = `${command}-settle-race-${after}`;
const threadId = ThreadId.make(`thread:${race}`);
const runId = RunId.make(`run:${race}`);
const taskId = NodeId.make(`node:${race}-task`);
const checkpointId = CheckpointId.make(`checkpoint:${race}`);
const messageId = MessageId.make(`message:delegated-delivery:${threadId}`);
const commandType = {
acknowledge: "delegated_task.completion-delivery.acknowledge",
accept: "notification.delivery.accept",
"wake-policy": "delegated_task.wake-policy",
}[command];
yield* seedParentWithTerminalTask({
threadId,
projectId: ProjectId.make(`project:${race}`),
runId,
rootNodeId: NodeId.make(`node:${race}-root`),
taskId,
...(command === "wake-policy"
? { deliveryState: "pending" }
: { deliveryState: "claimed", deliveryTaskIds: [taskId] }),
now,
});
const parent = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
yield* sink.write({
events: [
{
id: EventId.make(`event:${race}-before`),
type: "run.updated",
threadId,
runId,
occurredAt: now,
payload: { ...parent, status: before },
},
{
id: EventId.make(`event:${race}-message`),
type: "message.updated",
threadId,
runId,
occurredAt: now,
payload: {
id: messageId,
threadId,
runId,
nodeId: parent.rootNodeId,
role: "user",
text: "Background task finished",
attachments: [],
streaming: false,
createdBy: "agent",
creationSource: "server",
createdAt: now,
updatedAt: now,
delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [taskId] },
},
},
],
});
const { delegatedCompletion: _cohort, ...settledRun } = parent;
const commit = sink.commitCommand;
let settled = false;
let planned: OrchestrationV2Run["delegatedCompletion"];
yield* Effect.acquireUseRelease(
Effect.sync(() =>
vi.spyOn(sink, "commitCommand").mockImplementation((input) =>
Effect.gen(function* () {
if (!settled && input.commandType === commandType) {
settled = true;
const plannedRun = input.events.find((event) => event.type === "run.updated");
planned =
plannedRun?.type === "run.updated"
? plannedRun.payload.delegatedCompletion
: undefined;
// The provider turn ends (or its checkpoint lands) after the command read the run.
yield* commit({
commandId: CommandId.make(`command:${race}`),
threadId,
commandType: capture ? "checkpoint.capture" : "provider.event",
acceptedAt: now,
effects: [],
events: [
{
id: EventId.make(`event:${race}-settled`),
type: "run.updated",
threadId,
runId,
occurredAt: now,
payload: {
...settledRun,
status: after,
completedAt: now,
checkpointId: capture ? checkpointId : null,
},
},
],
});
}
return yield* commit(input);
}),
),
),
() =>
orchestrator.dispatch(
command === "accept"
? {
type: "notification.delivery.accept",
commandId: CommandId.make(`command:${race}:accept`),
threadId,
messageId,
}
: command === "wake-policy"
? {
type: "delegated_task.wake-policy",
commandId: CommandId.make(`command:${race}:wake-policy`),
parentThreadId: threadId,
taskId,
completionWake: "always",
}
: {
type: "delegated_task.completion-delivery.acknowledge",
commandId: CommandId.make(`command:${race}:task-status`),
parentThreadId: threadId,
taskId,
observedByRunId: runId,
},
),
(spy) => Effect.sync(() => spy.mockRestore()),
);
assert.isTrue(settled);
const finished = yield* orchestrator.getThreadProjection(threadId);
assert.equal(
finished.subagents.find((row) => row.id === taskId)?.completionDelivery?.state,
{ acknowledge: "acknowledged", accept: "delivered", "wake-policy": "claimed" }[command],
);
const run = finished.runs.find((candidate) => candidate.id === runId)!;
assert.deepEqual(
{
status: run.status,
completedAt: run.completedAt,
checkpointId: run.checkpointId,
delegatedCompletion: run.delegatedCompletion,
},
{
status: after,
completedAt: now,
checkpointId: capture ? checkpointId : null,
delegatedCompletion: planned,
},
);
}),
);
});
const reconcileCommandId = (name: string) => CommandId.make(`command:runtime-reconcile:${name}`);
const parentProviderThreadId = (threadId: ThreadId) =>
ProviderThreadId.make(`provider-thread:${String(threadId).replace("thread:", "")}`);
const runEvent = (input: {
readonly threadId: ThreadId;
readonly runId: RunId;
readonly ordinal: number;
readonly status: OrchestrationV2Run["status"];
readonly now: DateTime.Utc;
readonly providerThreadId?: ProviderThreadId;
readonly userMessageId?: MessageId;
readonly delegatedCompletion?: OrchestrationV2Run["delegatedCompletion"];
}) => ({
id: EventId.make(`event:${input.runId}:${input.status}`),
type: "run.updated" as const,
threadId: input.threadId,
runId: input.runId,
providerInstanceId: modelSelection.instanceId,
occurredAt: input.now,
payload: {
id: input.runId,
threadId: input.threadId,
ordinal: input.ordinal,
providerInstanceId: modelSelection.instanceId,
modelSelection,
providerThreadId: input.providerThreadId ?? null,
userMessageId: input.userMessageId ?? MessageId.make(`message:${input.runId}`),
rootNodeId: null,
activeAttemptId: null,
status: input.status,
requestedAt: input.now,
startedAt: input.now,
completedAt: input.status === "running" ? null : input.now,
checkpointId: null,
contextHandoffId: null,
...(input.delegatedCompletion === undefined
? {}
: { delegatedCompletion: input.delegatedCompletion }),
},
});
const seedRestartCancelledChild = (input: {
readonly parentThreadId: ThreadId;
readonly projectId: ProjectId;
readonly parentRunId: RunId;
readonly rootNodeId: NodeId;
readonly name: string;
readonly completionWake: "always" | "settled_only";
readonly continuationPending: boolean;
readonly runStatus?: "cancelled" | "completed";
readonly lostWork?: OrchestrationV2Run["restartCancelledBackgroundWork"];
readonly now: DateTime.Utc;
}) =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const taskId = NodeId.make(`node:${input.name}`);
const childThreadId = ThreadId.make(`thread:${input.name}`);
const childRunId = RunId.make(`run:${input.name}:1`);
yield* eventSink.write({
commandId: CommandId.make(`command:seed-child:${input.name}`),
events: [
{
id: EventId.make(`event:${input.name}:thread`),
type: "thread.created",
threadId: childThreadId,
occurredAt: input.now,
payload: {
createdBy: "agent",
creationSource: "server",
id: childThreadId,
projectId: input.projectId,
title: input.name,
providerInstanceId: modelSelection.instanceId,
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
activeProviderThreadId: null,
lineage: {
parentThreadId: input.parentThreadId,
relationshipToParent: "subagent",
rootThreadId: input.parentThreadId,
},
forkedFrom: { type: "node", nodeId: taskId },
createdAt: input.now,
updatedAt: input.now,
archivedAt: null,
settledOverride: null,
settledAt: null,
lastVisitedAt: null,
deletedAt: null,
},
},
{
id: EventId.make(`event:${input.name}:task`),
type: "subagent.updated",
threadId: input.parentThreadId,
runId: input.parentRunId,
nodeId: taskId,
driver,
providerInstanceId: modelSelection.instanceId,
occurredAt: input.now,
payload: {
id: taskId,
threadId: input.parentThreadId,
runId: input.parentRunId,
parentNodeId: input.rootNodeId,
origin: "app_owned",
createdBy: "agent",
driver,
providerInstanceId: modelSelection.instanceId,
providerThreadId: null,
childThreadId,
nativeTaskRef: null,
prompt: `Run ${input.name}.`,
title: null,
model: null,
completionWake: input.completionWake,
status: "running",
result: null,
startedAt: input.now,
completedAt: null,
updatedAt: input.now,
},
},
],
});
const childRun = runEvent({
threadId: childThreadId,
runId: childRunId,
ordinal: 1,
status: input.runStatus ?? "cancelled",
now: input.now,
});
yield* eventSink.writeWithEffects({
commandId: reconcileCommandId(input.name),
events: [
input.lostWork === undefined
? childRun
: {
...childRun,
payload: { ...childRun.payload, restartCancelledBackgroundWork: input.lostWork },
},
],
effects: input.continuationPending
? [
{
id: `effect:restart-continuation:${childRunId}`,
commandId: reconcileCommandId(input.name),
threadId: childThreadId,
request: { type: "provider-runtime.continue", sourceRunId: childRunId },
},
]
: [],
});
return { taskId, childThreadId, childRunId };
});
it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
it.effect.each([
{ name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
{ name: "Stop disposes", holdQueue: true, siblingFinishesFirst: false },
{ name: "an interrupt reserves", holdQueue: false, siblingFinishesFirst: true },
])("$name a completion wake's other siblings", ({ holdQueue, siblingFinishesFirst }) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const key = holdQueue
? "sibling-stop"
: siblingFinishesFirst
? "sibling-finished-interrupt"
: "sibling-interrupt";
const threadId = ThreadId.make(`${key}-parent`);
const originRunId = RunId.make(`${key}-origin`);
const incomingTaskId = NodeId.make(`${key}-incoming`);
yield* seedParentWithTerminalTask({
threadId,
projectId: ProjectId.make(`${key}-project`),
runId: originRunId,
rootNodeId: NodeId.make(`${key}-origin-root`),
taskId: incomingTaskId,
deliveryState: "claimed",
deliveryTaskIds: [incomingTaskId],
completionWake: "always",
now,
});
const originRootId = NodeId.make(`${key}-origin-root`);
yield* sink.write({
events: [
{
id: EventId.make(`event:${key}-origin-root`),
type: "node.updated",
threadId,
runId: originRunId,
nodeId: originRootId,
occurredAt: now,
payload: {
id: originRootId,
threadId,
runId: originRunId,
parentNodeId: null,
rootNodeId: originRootId,
kind: "root_turn",
status: "running",
countsForRun: true,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
runtimeRequestId: null,
checkpointScopeId: null,
startedAt: now,
completedAt: null,
},
},
],
});
yield* orchestrator.dispatch({
type: "delegated_task.request",
createdBy: "agent",
creationSource: "mcp",
commandId: CommandId.make(`${key}-sibling`),
parentThreadId: threadId,
parentRunId: originRunId,
parentNodeId: originRootId,
task: "Keep working",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
completionWake: "always",
});
const seeded = yield* orchestrator.getThreadProjection(threadId);
const origin = seeded.runs.find((run) => run.id === originRunId)!;
const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
yield* sink.write({
events: [
{
...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
payload: { ...origin, status: "completed", completedAt: now },
},
],
});
const delivery = origin.delegatedCompletion!.delivery!;
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "agent",
creationSource: "server",
commandId: CommandId.make(`${key}-wake`),
threadId,
messageId: delivery.messageId,
text: "Delegated task completed",
attachments: [],
modelSelection,
dispatchMode: { type: "queue_after_active" },
delegatedCompletion: {
parentRunId: originRunId,
generation: delivery.generation,
taskIds: delivery.taskIds,
},
});
yield* orchestrator.resumeQueuedRuns;
const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
(run) => run.userMessageId === delivery.messageId,
)!;
const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
const completeSibling = Effect.gen(function* () {
const afterSequence = yield* sink.latestSequence();
yield* sink.write({
events: [
{
...runEvent({
threadId: sibling.childThreadId!,
runId: childRun.id,
ordinal: childRun.ordinal,
status: "completed",
now,
}),
payload: { ...childRun, status: "completed", completedAt: now },
},
],
});
return yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
Stream.filter(
(stored) =>
stored.event.type === "subagent.updated" &&
stored.event.payload.id === sibling.id &&
stored.event.payload.status === "completed",
),
Stream.take(1),
Stream.runDrain,
Effect.as(afterSequence),
);
});
if (siblingFinishesFirst) {
yield* completeSibling;
assert.equal(
(yield* orchestrator.getThreadProjection(threadId)).subagents.find(
(task) => task.id === sibling.id,
)?.completionDelivery?.state,
"pending",
);
}
const beforeInterrupt = yield* sink.latestSequence();
const interrupted = yield* orchestrator.dispatch({
type: "run.interrupt",
commandId: CommandId.make(`${key}-interrupt`),
threadId,
runId: wake.id,
...(holdQueue ? { holdQueue } : {}),
});
// The wake's terminal may claim the sibling right after commit, so read what the interrupt
// itself wrote rather than a later projection.
const interruptDelivery = new Map(
interrupted.storedEvents.flatMap(({ event }) =>
event.type === "subagent.updated"
? [[event.payload.id, event.payload.completionDelivery?.state] as const]
: [],
),
);
const interruptCohort = interrupted.storedEvents.flatMap(({ event }) =>
event.type === "run.updated" && event.payload.id === originRunId
? [event.payload.delegatedCompletion]
: [],
);
assert.equal(interruptDelivery.get(incomingTaskId), "disposed");
assert.equal(interruptCohort.at(-1)?.disposition, holdQueue ? "stopped" : "open");
if (holdQueue) {
assert.equal(interruptDelivery.get(sibling.id), "disposed");
return;
}
assert.isFalse(interruptDelivery.has(sibling.id));
const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield* completeSibling;
// The wake's terminal, or the sibling's own completion, reserves the successor.
yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
Stream.filter(
(stored) =>
stored.event.type === "subagent.updated" &&
stored.event.payload.id === sibling.id &&
stored.event.payload.completionDelivery?.state === "claimed",
),
Stream.take(1),
Stream.runDrain,
);
const woken = yield* orchestrator.getThreadProjection(threadId);
const successor = woken.runs.find((run) => run.id === originRunId)?.delegatedCompletion
?.delivery;
assert.equal(successor?.generation, delivery.generation + 1);
assert.deepEqual(successor?.taskIds, [sibling.id]);
assert.notEqual(successor?.messageId, delivery.messageId);
// ProviderContinuationService consumes the claim's offer and dispatches the successor.
yield* sink.stream({ afterSequence, eventType: "message.updated" }).pipe(
Stream.filter(
(stored) =>
stored.event.type === "message.updated" &&
stored.event.payload.id === successor!.messageId,
),
Stream.take(1),
Stream.runDrain,
);
// Repeated recovery re-offers every open delivery; none may wake the parent twice.
yield* orchestrator.resumeQueuedRuns;
yield* orchestrator.recoverDelegatedTasks;
yield* orchestrator.recoverDelegatedTasks;
yield* orchestrator.resumeQueuedRuns;
const delivered = yield* orchestrator.getThreadProjection(threadId);
const cohort = delivered.runs.find((run) => run.id === originRunId)?.delegatedCompletion;
assert.equal(cohort?.disposition, "open");
assert.equal(cohort?.nextGeneration, delivery.generation + 2);
assert.deepEqual(cohort?.delivery, successor);
assert.deepEqual(
delivered.messages
.filter((message) => message.delegatedCompletion?.parentRunId === originRunId)
.map((message) => [message.id, message.delegatedCompletion]),
[
[
delivery.messageId,
{
parentRunId: originRunId,
generation: delivery.generation,
taskIds: [incomingTaskId],
},
],
[
successor!.messageId,
{ parentRunId: originRunId, generation: successor!.generation, taskIds: [sibling.id] },
],
],
);
const successorRuns = delivered.runs.filter(
(run) => run.userMessageId === successor!.messageId,
);
assert.equal(successorRuns.length, 1);
assert.equal(successorRuns[0]?.status, "starting");
assert.equal(
delivered.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
"claimed",
);
}),
);
});
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
"subagent follow-ups",
(it) => {
it.effect("recovers a reserved delivery without loading conversation history", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const sink = yield* EventSink.EventSinkV2;
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:bounded-delivery-recovery");
const runId = RunId.make("run:bounded-delivery-recovery");
const taskId = NodeId.make("task:bounded-delivery-recovery");
yield* seedParentWithTerminalTask({
threadId,
runId,
taskId,
projectId: ProjectId.make("project:bounded-delivery-recovery"),
rootNodeId: NodeId.make("node:bounded-delivery-recovery"),
deliveryState: "claimed",
deliveryTaskIds: [taskId],
completionWake: "always",
now,
});
for (const suffix of ["first", "second"]) {
yield* sink.write({
events: [
{
id: EventId.make(`event:bounded-delivery-history:${suffix}`),
type: "message.updated",
threadId,
occurredAt: now,
payload: {
id: MessageId.make(`message:bounded-delivery-history:${suffix}`),
threadId,
runId: null,
nodeId: null,
createdBy: "user",
creationSource: "web",
role: "user",
text: "Unrelated historical input",
attachments: [],
streaming: false,
createdAt: now,
updatedAt: now,
},
},
],
});
}
const statements: string[] = [];
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (time, exit) => {
end(time, exit);
const query = span.attributes.get("db.query.text");
if (typeof query === "string") statements.push(query);
};
return span;
},
});
yield* orchestrator.recoverDelegatedTasks.pipe(Effect.withTracer(tracer));
assert.isFalse(statements.some((query) => query.includes("MIN(ordinal)")));
const messageReads = statements.filter((query) =>
query.includes("FROM orchestration_v2_projection_messages"),
);
assert.isNotEmpty(messageReads);
assert.isTrue(messageReads.every((query) => query.includes("message_id IN")));
const wake = yield* requests.take;
assert.equal(wake.delegatedCompletion?.parentRunId, runId);
}),
);