import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
import { SourceControlProviderKind } from "@t3tools/contracts";
import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";
import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
import { vi } from "vite-plus/test";
import {
type ApplicationStoredEvent,
CheckpointId,
CheckpointRef,
CommandId,
ContextTransferId,
EventId,
MessageId,
NodeId,
RuntimeRequestId,
TurnItemId,
type ModelSelection,
type OrchestrationV2Run,
type OrchestrationV2ShellSnapshot,
type OrchestrationV2RuntimeRequest,
ProjectId,
type PullRequestDetail,
type PullRequestComment,
PullRequestOperationError,
ProviderDriverKind,
ProviderInstanceId,
ProviderSessionId,
ProviderThreadId,
ProviderTurnId,
RunId,
ScheduledTaskId,
ThreadId,
} from "@t3tools/contracts";
import * as Effect from "effect/Effect";
import * as DateTime from "effect/DateTime";
import * as Deferred from "effect/Deferred";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Queue from "effect/Queue";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import * as SqlClient from "effect/sql/SqlClient";
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
import * as GitWorkflow from "../git/GitWorkflowService.ts";
import * as ServerConfig from "../config.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as OrchestrationEventStore from "../persistence/OrchestrationEventStore.ts";
import * as ProjectEnrichmentService from "../project/ProjectEnrichmentService.ts";
import * as ProjectService from "../project/ProjectService.ts";
import * as ServerSettings from "../serverSettings.ts";
import * as McpSessionRegistryTestkit from "../mcp/McpSessionRegistry.testkit.ts";
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
import * as VcsProcess from "../vcs/VcsProcess.ts";
import * as WorkspacePaths from "../workspace/WorkspacePaths.ts";
import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts";
import * as Orchestrator from "./Orchestrator.ts";
import { ROLLBACK_FAILED_MESSAGE } from "./CheckpointRollbackService.ts";
import * as EffectWorker from "./EffectWorker.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import * as EventSink from "./EventSink.ts";
import * as ProviderRuntimeRecoveryService from "./ProviderRuntimeRecoveryService.ts";
import * as EventStore from "./EventStore.ts";
import * as ProjectionReplay from "./testkit/ProjectionReplay.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as PullRequestWatchReactor from "./PullRequestWatchReactor.ts";
import { PULL_REQUEST_WATCH_WAKE_LIMIT } from "./pullRequestWatch.ts";
import * as PullRequestService from "../pullRequest/PullRequestService.ts";
import * as ProjectStore from "./ProjectStore.ts";
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
import * as RuntimeLayer from "./runtimeLayer.ts";
import { shellStreamItemFromThreadShell } from "./ShellStream.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
import * as ThreadManagementService from "./ThreadManagementService.ts";
import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
import * as ScratchThreadPromotion from "../project/ScratchThreadPromotion.ts";
import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts";
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
import * as Scheduler from "../scheduling/Scheduler.ts";
import * as SecretRequests from "../secrets/SecretRequests.ts";
import * as PreviewManager from "../preview/Manager.ts";
import * as TerminalManager from "../terminal/Manager.ts";
import * as ResourceCleanupService from "./ResourceCleanupService.ts";
import * as ThreadLaunchService from "./ThreadLaunchService.ts";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
const layerPlatformTest = Layer.merge(
Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({
resolveLink: () => Effect.die("unused title link"),
}),
);
const layerServerConfig = ServerConfig.layerTest(process.cwd(), {
prefix: "t3-orchestration-v2-runtime-layer-",
});
const modelSelection = {
instanceId: ProviderInstanceId.make("codex"),
model: "gpt-5.4",
} satisfies ModelSelection;
const alternateInstanceId = ProviderInstanceId.make("codex_alternate");
const layerVcsDriverRegistryTest = VcsDriverRegistry.layer.pipe(
Layer.provide(VcsProcess.layer),
Layer.provide(layerServerConfig),
Layer.provide(layerPlatformTest),
);
const layerCheckpointStoreTest = CheckpointStore.layer.pipe(
Layer.provide(layerVcsDriverRegistryTest),
);
const layerGitWorkflowTest = Layer.mock(GitWorkflow.GitWorkflowService)({
pruneWorktrees: () => Effect.void,
createWorktree: () => Effect.succeed({} as never),
});
const layerProjectServiceTest = Layer.mock(ProjectService.ProjectService)({
getById: () => Effect.succeedNone,
});
const driver = ProviderDriverKind.make("codex");
const orchestrationAdapter = {
instanceId: modelSelection.instanceId,
driver,
getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }),
openSession: () => Effect.die("sessions are not used by lifecycle tests"),
} as ProviderAdapter.ProviderAdapterV2["Service"];
const providerInstance = {
instanceId: modelSelection.instanceId,
driverKind: driver,
continuationIdentity: {
driverKind: driver,
continuationKey: "codex:test",
},
displayName: "Codex test",
enabled: true,
snapshot: { getSnapshot: Effect.succeed({}) } as unknown as ProviderInstance["snapshot"],
orchestrationAdapter,
textGeneration: {} as ProviderInstance["textGeneration"],
} satisfies ProviderInstance;
const alternateProviderInstance = {
...providerInstance,
instanceId: alternateInstanceId,
continuationIdentity: {
driverKind: driver,
continuationKey: "codex:test:alternate",
},
displayName: "Codex alternate test",
orchestrationAdapter: {
...orchestrationAdapter,
instanceId: alternateInstanceId,
},
} satisfies ProviderInstance;
const layerTestProviderInstanceRegistry = Layer.succeed(
ProviderInstanceRegistry.ProviderInstanceRegistry,
{
getInstance: (instanceId) =>
Effect.succeed(
[providerInstance, alternateProviderInstance].find(
(instance) => instanceId === instance.instanceId,
),
),
listInstances: Effect.succeed([providerInstance, alternateProviderInstance]),
listUnavailable: Effect.succeed([]),
streamChanges: Stream.empty,
subscribeChanges: Effect.never,
},
);
const watchedPullRequestDetail = (input: {
readonly projectId: ProjectId;
readonly number: number;
readonly at: string;
}): PullRequestDetail => ({
provider: SourceControlProviderKind.make("github"),
capabilities: {
diff: true,
comment: true,
actions: [],
mergeMethods: [],
search: false,
checksAlwaysReport: true,
review: { inlineComment: false, reply: false, resolve: false, verdicts: [] },
reviewers: { request: false, listCandidates: false },
},
viewerPermissions: {
actions: [],
comment: true,
resolve: true,
verdicts: [],
requestReviewers: false,
},
projectId: input.projectId,
projectTitle: "Watch wake",
workspaceRoot: "/workspace/watch",
repository: "pingdotgg/t3code",
number: input.number,
title: "Watched pull request",
body: "",
url: `https://github.com/pingdotgg/t3code/pull/${input.number}`,
author: { login: "agent-user", name: null, avatarUrl: null },
state: "open",
isDraft: false,
mergeability: "mergeable",
additions: 1,
deletions: 0,
changedFiles: 1,
headBranch: "feature",
headSha: "abc1234def",
baseBranch: "main",
createdAt: input.at,
updatedAt: input.at,
mergedAt: null,
closedAt: null,
reviewers: [],
labels: [],
checks: [{ name: "lint", status: "failure", description: null, url: null }],
mergeCapabilities: { merge: true, squash: true, rebase: true },
viewer: "agent-user",
});
const seedProject = (input: {
readonly projectId: ProjectId;
readonly title: string;
readonly workspaceRoot: string;
readonly defaultModelSelection: ModelSelection | null;
readonly createdAt: string;
}) =>
Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) =>
projects.apply({
sequence: 0,
eventId: EventId.make(`seed:${input.projectId}`),
aggregateKind: "project",
aggregateId: input.projectId,
occurredAt: input.createdAt,
commandId: null,
causationEventId: null,
correlationId: null,
metadata: {},
type: "project.created",
payload: {
projectId: input.projectId,
title: input.title,
workspaceRoot: input.workspaceRoot,
defaultModelSelection: input.defaultModelSelection,
scripts: [],
createdAt: input.createdAt,
updatedAt: input.createdAt,
},
}),
);
const moveProject = (projectId: ProjectId, workspaceRoot: string, updatedAt: string) =>
Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) =>
projects.apply({
sequence: 0,
eventId: EventId.make(`move:${projectId}:${workspaceRoot}`),
aggregateKind: "project",
aggregateId: projectId,
occurredAt: updatedAt,
commandId: null,
causationEventId: null,
correlationId: null,
metadata: {},
type: "project.meta-updated",
payload: { projectId, workspaceRoot, updatedAt },
}),
);
// Read-only oracle: projections still equal a replay of these unpruned test logs.
const layerProjectionReplayTest = ProjectionReplay.layer.pipe(
Layer.provide(
Layer.merge(
EventStore.layerFromOrchestrationEventStore.pipe(
Layer.provide(RuntimeLayer.layerEventInfrastructure),
),
ProjectionStore.layer,
),
),
);
const layerTest = Layer.mergeAll(
RuntimeLayer.layer,
layerProjectionReplayTest,
RuntimeLayer.layerEventSink,
ProjectStore.layer,
ProjectionStore.layer,
EffectOutbox.layer,
ThreadCommandExecutor.layer,
).pipe(
Layer.provide(McpSessionRegistryTestkit.layer),
Layer.provide(McpProviderSessions.layer),
Layer.provide(SqlitePersistence.layerMemory),
Layer.provide(layerCheckpointStoreTest),
Layer.provide(layerServerConfig),
Layer.provide(ServerSettings.layerTest()),
Layer.provide(layerTestProviderInstanceRegistry),
Layer.provide(layerGitWorkflowTest),
Layer.provide(layerProjectServiceTest),
Layer.provide(layerPlatformTest),
);
const layerLegacyImportTest = Layer.merge(RuntimeLayer.layer, layerProjectionReplayTest).pipe(
Layer.provide(McpSessionRegistryTestkit.layer),
Layer.provide(McpProviderSessions.layer),
Layer.provideMerge(SqlitePersistence.layerMemory),
Layer.provide(layerCheckpointStoreTest),
Layer.provide(layerServerConfig),
Layer.provide(ServerSettings.layerTest()),
Layer.provide(layerTestProviderInstanceRegistry),
Layer.provide(layerGitWorkflowTest),
Layer.provide(layerProjectServiceTest),
Layer.provide(layerPlatformTest),
);
const layerScheduledTasksTest = ScheduledTaskService.layer.pipe(
Layer.provide(
Layer.mergeAll(
ManagedPolicy.layerTest(),
Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
Layer.mock(ThreadManagementService.ThreadManagementService)({}),
Layer.mock(SecretRequests.SecretRequests)({}),
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
namedProjectsRoot: "/projects",
isScratchProject: () => Effect.succeed(false),
}),
Scheduler.layer,
),
),
);
const layerResourceCleanupTest = ResourceCleanupService.layer.pipe(
Layer.provideMerge(layerScheduledTasksTest),
Layer.provide(
Layer.mergeAll(
Layer.mock(PreviewManager.PreviewManager)({ close: () => Effect.void }),
Layer.mock(TerminalManager.TerminalManager)({ close: () => Effect.void }),
),
),
);
const layerProjectDeletionTest = Layer.mergeAll(
RuntimeLayer.layer.pipe(Layer.provide(RuntimeLayer.layerProjectService)),
RuntimeLayer.layerProjectService,
RuntimeLayer.layerEventSink,
ThreadCommandExecutor.layer,
).pipe(
Layer.provide(
Layer.mock(ProjectEnrichmentService.ProjectEnrichmentService)({
peek: () =>
Effect.succeed({
repositoryIdentity: null,
faviconPath: null,
repositoryIdentityResolved: false,
}),
getAvailable: () =>
Effect.succeed({
repositoryIdentity: null,
faviconPath: null,
repositoryIdentityResolved: false,
}),
invalidate: () => Effect.void,
}),
),
Layer.provide(
Layer.mock(WorkspacePaths.WorkspacePaths)({
normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(workspaceRoot),
}),
),
Layer.provideMerge(layerResourceCleanupTest),
Layer.provide(McpSessionRegistryTestkit.layer),
Layer.provide(McpProviderSessions.layer),
Layer.provide(SqlitePersistence.layerMemory),
Layer.provide(layerCheckpointStoreTest),
Layer.provide(layerServerConfig),
Layer.provide(ServerSettings.layerTest()),
Layer.provide(layerTestProviderInstanceRegistry),
Layer.provide(layerGitWorkflowTest),
Layer.provide(layerPlatformTest),
);
it.layer(layerProjectDeletionTest)("project deletion during thread commands", (it) => {
it.effect.each(["thread", "project"] as const)(
"deleting a %s removes its thread-bound automations through the effect worker",
(deletion) =>
Effect.gen(function* () {
const projects = yield* ProjectService.ProjectService;
const orchestrator = yield* Orchestrator.OrchestratorV2;
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
const tasks = yield* ScheduledTaskService.ScheduledTaskService;
const projectId = ProjectId.make(`automation-delete-project:${deletion}`);
const threadId = ThreadId.make(`automation-delete-thread:${deletion}`);
const otherThreadId = ThreadId.make(`automation-surviving-thread:${deletion}`);
yield* projects.create({
commandId: CommandId.make(`automation-create-project:${deletion}`),
projectId,
title: "Automation lifecycle",
workspaceRoot: `/work/automation-lifecycle/${deletion}`,
});
for (const id of [threadId, otherThreadId]) {
yield* orchestrator.dispatch({
type: "thread.create",
commandId: CommandId.make(`automation-create:${id}`),
createdBy: "user",
creationSource: "web",
threadId: id,
projectId: id === threadId ? projectId : ProjectId.make("automation-other-project"),
title: "Automation target",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
});
}
const save = (name: string, target: ThreadId | null, webhook = false) =>
tasks.upsert({
// As schedule_task sends it, so clearing the thread unbinds the task.
threadMode: target === null ? "new_thread" : "continue",
id: ScheduledTaskId.make(`automation:${deletion}:${name}`),
title: name,
prompt: "Check orchestration progress",
enabled: !webhook,
schedule: webhook ? { type: "webhook" } : { type: "interval", everyMs: 60_000 },
projectId,
threadId: target,
workspaceStrategy: { type: "root" },
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
createdBy: "agent",
creationSource: "mcp",
});
const { task: bound } = yield* save("bound", threadId);
const { task: webhook } = yield* save("webhook", threadId, true);
const { task: independent } = yield* save("independent", null);
yield* save("detached", threadId);
const { task: detached } = yield* save("detached", null);
yield* save("moved", threadId);
const { task: moved } = yield* save("moved", otherThreadId);
const { task: other } = yield* save("other", otherThreadId);
yield* tasks.triggerWebhook({
hookId: webhook.id,
token: webhook.webhook!.path.split("/").at(-1)!,
method: "POST",
path: webhook.webhook!.path,
query: "",
headers: {},
body: new Uint8Array(),
bodyText: "",
});
assert.lengthOf((yield* tasks.listWebhookDeliveries({ id: webhook.id })).deliveries, 1);
yield* orchestrator.dispatch({
type: "thread.archive",
commandId: CommandId.make(`automation-archive:${deletion}`),
threadId,
});
yield* worker.drain();
assert.include(
(yield* tasks.list()).tasks.map((task) => task.id),
bound.id,
);
if (deletion === "thread") {
yield* orchestrator.dispatch({
type: "thread.delete",
commandId: CommandId.make(`automation-delete:${deletion}`),
threadId,
});
} else {
yield* projects.delete({
commandId: CommandId.make(`automation-delete:${deletion}`),
projectId,
force: true,
});
}
yield* worker.drain();
const remainingIds = (yield* tasks.list()).tasks.map((task) => task.id);
assert.notInclude(remainingIds, bound.id);
assert.notInclude(remainingIds, webhook.id);
assert.includeMembers(remainingIds, [independent.id, detached.id, moved.id, other.id]);
assert.lengthOf((yield* tasks.listWebhookDeliveries({ id: webhook.id })).deliveries, 0);
const rejected = yield* tasks.runNow({ id: bound.id }).pipe(Effect.flip);
assert.equal(rejected.message, "Schedule task not found.");
assert.equal(
(yield* tasks.triggerWebhook({
hookId: webhook.id,
token: webhook.webhook!.path.split("/").at(-1)!,
method: "POST",
path: webhook.webhook!.path,
query: "",
headers: {},
body: new Uint8Array(),
bodyText: "",
}))._tag,
"not_found",
);
// Retried cleanup is harmless and cannot affect unrelated tasks.
yield* tasks.deleteForThread(threadId);
assert.sameMembers(
(yield* tasks.list()).tasks.map((task) => task.id),
remainingIds,
);
}),
);
it.effect("waits for an in-flight thread update before planning deletion", () =>
Effect.gen(function* () {
const projects = yield* ProjectService.ProjectService;
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
const projectId = ProjectId.make("runtime-project-delete-concurrent");
const threadId = ThreadId.make("runtime-thread-delete-concurrent");
const updateCommandId = CommandId.make("runtime-thread-delete-concurrent-update");
yield* projects.create({
commandId: CommandId.make("runtime-project-delete-concurrent-create"),
projectId,
title: "Concurrent deletion",
workspaceRoot: "/work/concurrent-deletion",
});
yield* orchestrator.dispatch({
type: "thread.create",
commandId: CommandId.make("runtime-thread-delete-concurrent-create"),
createdBy: "user",
creationSource: "web",
threadId,
projectId,
title: "Original title",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
});
const updateReady = yield* Deferred.make<void>();
const releaseUpdate = yield* Deferred.make<void>();
const deletionQueued = yield* Deferred.make<void>();
const commitCommand = eventSink.commitCommand;
const withLock = executor.withLock;
let threadLockRequests = 0;
const commitSpy = vi
.spyOn(eventSink, "commitCommand")
.mockImplementation((input) =>
input.commandId === updateCommandId
? Deferred.succeed(updateReady, undefined).pipe(
Effect.andThen(Deferred.await(releaseUpdate)),
Effect.andThen(commitCommand(input)),
)
: commitCommand(input),
);
const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
key,
effect,
) => {
if (key !== threadId || ++threadLockRequests !== 2) return withLock(key, effect);
return Deferred.succeed(deletionQueued, undefined).pipe(
Effect.andThen(withLock(key, effect)),
);
};
const lockSpy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
yield* Effect.gen(function* () {
const updateFiber = yield* orchestrator
.dispatch({
type: "thread.metadata.update",
commandId: updateCommandId,
threadId,
title: "Updated before deletion",
})
.pipe(Effect.forkChild({ startImmediately: true }));
yield* Effect.raceFirst(
Deferred.await(updateReady),
Fiber.join(updateFiber).pipe(
Effect.andThen(Effect.die("The update completed before reaching its commit barrier.")),
),
);
const deleteFiber = yield* projects
.delete({
commandId: CommandId.make("runtime-project-delete-concurrent-delete"),
projectId,
force: true,
})
.pipe(Effect.forkChild({ startImmediately: true }));
yield* Effect.raceFirst(
Deferred.await(deletionQueued),
Fiber.join(deleteFiber).pipe(
Effect.andThen(Effect.die("Project deletion bypassed the in-flight thread command.")),
),
);
yield* Deferred.succeed(releaseUpdate, undefined);
yield* Fiber.join(updateFiber);
const deletedProject = yield* Fiber.join(deleteFiber);
const projection = yield* orchestrator.getThreadProjection(threadId);
assert.isNotNull(deletedProject.deletedAt);
assert.isNotNull(projection.thread.deletedAt);
assert.equal(projection.thread.title, "Updated before deletion");
}).pipe(
Effect.ensuring(
Effect.sync(() => {
commitSpy.mockRestore();
lockSpy.mockRestore();
}),
),
);
}),
);
it.effect(
"deletes a side chat with its parent only after the side chat's in-flight command",
() =>
Effect.gen(function* () {
const projects = yield* ProjectService.ProjectService;
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
const projectId = ProjectId.make("runtime-side-chat-delete");
const parentId = ThreadId.make("runtime-side-chat-delete-parent");
const sideChatId = ThreadId.make("runtime-side-chat-delete-child");
const updateCommandId = CommandId.make("runtime-side-chat-delete-update");
yield* projects.create({
commandId: CommandId.make("runtime-side-chat-delete-project"),
projectId,
title: "Side chat deletion",
workspaceRoot: "/work/side-chat-deletion",
});
yield* orchestrator.dispatch({
type: "thread.create",
commandId: CommandId.make("runtime-side-chat-delete-create"),
createdBy: "user",
creationSource: "web",
threadId: parentId,
projectId,
title: "Parent",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
});
yield* orchestrator.dispatch({
type: "thread.fork",
commandId: CommandId.make("runtime-side-chat-delete-fork"),
sourceThreadId: parentId,
targetThreadId: sideChatId,
sourcePoint: { type: "latest_stable" },
sideChat: true,
createdBy: "user",
creationSource: "web",
});
const updateReady = yield* Deferred.make<void>();
const releaseUpdate = yield* Deferred.make<void>();
const deletionQueued = yield* Deferred.make<void>();
const commitCommand = eventSink.commitCommand;
const withLock = executor.withLock;
let sideChatLockRequests = 0;
const commitSpy = vi
.spyOn(eventSink, "commitCommand")
.mockImplementation((input) =>
input.commandId === updateCommandId
? Deferred.succeed(updateReady, undefined).pipe(
Effect.andThen(Deferred.await(releaseUpdate)),
Effect.andThen(commitCommand(input)),
)
: commitCommand(input),
);
const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
key,
effect,
) => {
if (key !== sideChatId || ++sideChatLockRequests !== 2) return withLock(key, effect);
return Deferred.succeed(deletionQueued, undefined).pipe(
Effect.andThen(withLock(key, effect)),
);
};
const lockSpy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
yield* Effect.gen(function* () {
const updateFiber = yield* orchestrator
.dispatch({
type: "thread.metadata.update",
commandId: updateCommandId,
threadId: sideChatId,
title: "Updated before deletion",
})
.pipe(Effect.forkChild({ startImmediately: true }));
yield* Effect.raceFirst(
Deferred.await(updateReady),
Fiber.join(updateFiber).pipe(
Effect.andThen(
Effect.die("The update completed before reaching its commit barrier."),
),
),
);
const deleteFiber = yield* orchestrator
.dispatch({
type: "thread.delete",
commandId: CommandId.make("runtime-side-chat-delete-parent"),
threadId: parentId,
})
.pipe(Effect.forkChild({ startImmediately: true }));
yield* Effect.raceFirst(
Deferred.await(deletionQueued),
Fiber.join(deleteFiber).pipe(
Effect.andThen(Effect.die("Parent deletion bypassed the side chat's command.")),
),
);
yield* Deferred.succeed(releaseUpdate, undefined);
yield* Fiber.join(updateFiber);
yield* Fiber.join(deleteFiber);
const sideChat = yield* orchestrator.getThreadProjection(sideChatId);
assert.isNotNull(sideChat.thread.deletedAt);
assert.equal(sideChat.thread.title, "Updated before deletion");
}).pipe(
Effect.ensuring(
Effect.sync(() => {
commitSpy.mockRestore();
lockSpy.mockRestore();
}),
),
);
}),
);
// A fork from a side chat onto its parent used to lock the side chat, then the parent: the
// reverse of the parent delete's order.
it.effect("a parent delete and a fork from its side chat onto the parent do not deadlock", () =>
Effect.gen(function* () {
const projects = yield* ProjectService.ProjectService;
const orchestrator = yield* Orchestrator.OrchestratorV2;
const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
const projectId = ProjectId.make("runtime-side-chat-lock-order");
const parentId = ThreadId.make("runtime-side-chat-lock-order-parent");
const sideChatId = ThreadId.make("runtime-side-chat-lock-order-child");
yield* projects.create({
commandId: CommandId.make("runtime-side-chat-lock-order-project"),
projectId,
title: "Side chat lock order",
workspaceRoot: "/work/side-chat-lock-order",
});
yield* orchestrator.dispatch({
type: "thread.create",
commandId: CommandId.make("runtime-side-chat-lock-order-create"),
createdBy: "user",
creationSource: "web",
threadId: parentId,
projectId,
title: "Parent",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
});
yield* orchestrator.dispatch({
type: "thread.fork",
commandId: CommandId.make("runtime-side-chat-lock-order-fork"),
sourceThreadId: parentId,
targetThreadId: sideChatId,
sourcePoint: { type: "latest_stable" },
sideChat: true,
createdBy: "user",
creationSource: "web",
});
const deleteHoldsParent = yield* Deferred.make<void>();
const releaseDelete = yield* Deferred.make<void>();
const forkWaitsForParent = yield* Deferred.make<void>();
const withLock = executor.withLock;
let parentLockRequests = 0;
const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
key,
effect,
) => {
if (key !== parentId) return withLock(key, effect);
// Dispatch builds locks it may not run, so count when one runs.
return Effect.suspend(() =>
++parentLockRequests === 1
? withLock(
key,
Deferred.succeed(deleteHoldsParent, undefined).pipe(
Effect.andThen(Deferred.await(releaseDelete)),
Effect.andThen(effect),
),
)
: Deferred.succeed(forkWaitsForParent, undefined).pipe(
Effect.andThen(withLock(key, effect)),
),
);
};
const lockSpy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
yield* Effect.gen(function* () {
const deleteFiber = yield* orchestrator
.dispatch({
type: "thread.delete",
commandId: CommandId.make("runtime-side-chat-lock-order-delete"),
threadId: parentId,
})
.pipe(Effect.forkChild({ startImmediately: true }));
yield* Deferred.await(deleteHoldsParent);
const forkFiber = yield* orchestrator
.dispatch({
type: "thread.fork",
commandId: CommandId.make("runtime-side-chat-lock-order-fork-back"),
sourceThreadId: sideChatId,
targetThreadId: parentId,
sourcePoint: { type: "latest_stable" },
sideChat: true,
createdBy: "user",
creationSource: "web",
})
.pipe(Effect.flip, Effect.forkChild({ startImmediately: true }));
yield* Deferred.await(forkWaitsForParent);
yield* Deferred.succeed(releaseDelete, undefined);
yield* Fiber.join(deleteFiber);
yield* Fiber.join(forkFiber);
assert.isNotNull((yield* orchestrator.getThreadProjection(parentId)).thread.deletedAt);
assert.isNotNull((yield* orchestrator.getThreadProjection(sideChatId)).thread.deletedAt);
}).pipe(Effect.ensuring(Effect.sync(() => lockSpy.mockRestore())));
}),
);
});
const layerSharedApplicationDataPlaneTest = Layer.mergeAll(
layerProjectionReplayTest,
RuntimeLayer.layer.pipe(Layer.provide(RuntimeLayer.layerProjectService)),
RuntimeLayer.layerProjectService,
RuntimeLayer.layerEventSink,
RuntimeLayer.layerEventInfrastructure,
).pipe(
Layer.provide(
Layer.mock(WorkspacePaths.WorkspacePaths)({
normalizeWorkspaceRoot: (workspaceRoot) => Effect.succeed(workspaceRoot),
}),
),
Layer.provide(
Layer.succeed(ProjectEnrichmentService.ProjectEnrichmentService, {
peek: () =>
Effect.succeed({
repositoryIdentity: null,
faviconPath: null,
repositoryIdentityResolved: false,
}),
request: () => Effect.void,
readRepositoryIdentity: () => Effect.succeed(null),
getAvailable: () =>
Effect.succeed({
repositoryIdentity: null,
faviconPath: null,
repositoryIdentityResolved: false,
}),
invalidate: () => Effect.void,
subscribeChanges: Effect.never,
}),
),
Layer.provide(McpSessionRegistryTestkit.layer),
Layer.provide(McpProviderSessions.layer),
Layer.provideMerge(SqlitePersistence.layerMemory),
Layer.provide(layerCheckpointStoreTest),
Layer.provide(layerServerConfig),
Layer.provide(ServerSettings.layerTest()),
Layer.provide(layerTestProviderInstanceRegistry),
Layer.provide(layerGitWorkflowTest),
Layer.provide(layerPlatformTest),
);
it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
const seedFailingRollbackThread = (name: string) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const threadId = ThreadId.make(name);
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make(`${name}-create`),
threadId,
projectId: ProjectId.make(`${name}-project`),
title: "Rollback failure",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: `/tmp/t3-${name}`,
});
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make(`${name}-message`),
threadId,
messageId: MessageId.make(`${name}-message`),
text: "Create the provider thread and checkpoint scope.",
attachments: [],
dispatchMode: { type: "start_immediately" },
});
const scope = (yield* orchestrator.getThreadProjection(threadId)).checkpointScopes[0]!;
const now = yield* DateTime.now;
const checkpointId = CheckpointId.make(`${name}-checkpoint`);
yield* eventSink.write({
commandId: CommandId.make(`${name}-seed`),
events: [
{
id: EventId.make(`${name}-checkpoint-event`),
type: "checkpoint.captured",
threadId,
occurredAt: now,
payload: {
id: checkpointId,
threadId,
scopeId: scope.id,
runId: null,
nodeId: scope.nodeId,
parentCheckpointId: null,
ordinalWithinScope: 0,
appRunOrdinal: null,
ref: CheckpointRef.make(`refs/t3/${name}`),
status: "ready",
files: [],
capturedAt: now,
},
},
],
});
yield* outbox.cancelUnsettled({
threadId,
effectTypes: ["provider-turn.start"],
reason: "not under test",
});
return {
threadId,
rollback: (commandId: CommandId) =>
orchestrator.dispatch({
type: "checkpoint.rollback",
commandId,
threadId,
checkpointId,
scopeId: scope.id,
restoreFiles: false,
}),
};
});
it.effect("projects a rollback that fails every attempt and clears it on the next one", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
const { threadId, rollback } = yield* seedFailingRollbackThread("runtime-rollback-failure");
const rollbackCommandId = CommandId.make("runtime-rollback-failure-rollback");
yield* rollback(rollbackCommandId);
for (let attempt = 0; attempt < 5; attempt++) {
yield* worker.drain();
yield* TestClock.adjust("30 seconds");
}
const [rollbackEffect] = yield* outbox.listByCommandId(rollbackCommandId);
assert.equal(rollbackEffect?.status, "failed");
const failed = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(failed.thread.rollbackFailure, {
requestId: rollbackCommandId,
message: ROLLBACK_FAILED_MESSAGE,
});
yield* rollback(CommandId.make("runtime-rollback-failure-retry"));
const retried = yield* orchestrator.getThreadProjection(threadId);
assert.isNull(retried.thread.rollbackFailure);
}),
);
it.effect("rejects non-ready rollback targets before persisting events or effects", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const threadId = ThreadId.make("runtime-rollback-readiness");
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make("runtime-rollback-readiness-create"),
threadId,
projectId: ProjectId.make("runtime-rollback-readiness-project"),
title: "Rollback readiness",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: process.cwd(),
});
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make("runtime-rollback-readiness-message"),
threadId,
messageId: MessageId.make("runtime-rollback-readiness-message"),
text: "Create the provider thread and checkpoint scope.",
attachments: [],
dispatchMode: { type: "start_immediately" },
});
const projection = yield* orchestrator.getThreadProjection(threadId);
const scope = projection.checkpointScopes[0]!;
const now = yield* DateTime.now;
for (const status of ["missing", "error", "stale", "ready"] as const) {
const checkpointId = CheckpointId.make("runtime-rollback-checkpoint");
const commandId = CommandId.make(`runtime-rollback-${status}`);
yield* eventSink.write({
commandId: CommandId.make(`runtime-rollback-${status}-seed`),
events: [
{
id: EventId.make(`runtime-rollback-${status}-event`),
type: "checkpoint.captured",
threadId,
occurredAt: now,
payload: {
id: checkpointId,
threadId,
scopeId: scope.id,
runId: null,
nodeId: scope.nodeId,
parentCheckpointId: null,
ordinalWithinScope: 0,
appRunOrdinal: null,
ref: CheckpointRef.make(`refs/t3/runtime-rollback-${status}`),
status,
files: [],
capturedAt: now,
},
},
],
});
const previousSequence = yield* orchestrator.getThreadEventSequence(threadId);
const rollback = orchestrator.dispatch({
type: "checkpoint.rollback",
commandId,
threadId,
checkpointId,
scopeId: scope.id,
});
if (status === "ready") {
const accepted = yield* rollback;
assert.deepEqual(
accepted.storedEvents.map((stored) => stored.event.type),
["thread.metadata-updated", "checkpoint.rollback-requested"],
);
assert.deepEqual(
(yield* outbox.listByCommandId(commandId)).map((effect) => effect.request.type),
["provider-thread.rollback"],
);
const path = yield* Path.Path.pipe(
Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
);
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make("runtime-rollback-ancestor-create"),
threadId: ThreadId.make("runtime-rollback-ancestor"),
projectId: ProjectId.make("runtime-rollback-readiness-project"),
title: "Ancestor workspace owner",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: path.dirname(process.cwd()),
});
const overlapCommandId = CommandId.make("runtime-rollback-overlap");
const overlapSequence = yield* orchestrator.getThreadEventSequence(threadId);
const overlap = yield* orchestrator
.dispatch({
type: "checkpoint.rollback",
commandId: overlapCommandId,
threadId,
checkpointId,
scopeId: scope.id,
})
.pipe(Effect.flip);
assert.match(String(overlap.cause), /isolated worktree/);
assert.equal(yield* orchestrator.getThreadEventSequence(threadId), overlapSequence);
assert.deepEqual(yield* outbox.listByCommandId(overlapCommandId), []);
yield* orchestrator.dispatch({
type: "thread.metadata.update",
commandId: CommandId.make("runtime-rollback-share"),
threadId,
worktreePath: null,
});
const sharedCommandId = CommandId.make("runtime-rollback-shared");
const sequence = yield* orchestrator.getThreadEventSequence(threadId);
const shared = yield* orchestrator
.dispatch({
type: "checkpoint.rollback",
commandId: sharedCommandId,
threadId,
checkpointId,
scopeId: scope.id,
})
.pipe(Effect.flip);
assert.match(String(shared.cause), /isolated worktree/);
assert.equal(yield* orchestrator.getThreadEventSequence(threadId), sequence);
assert.deepEqual(yield* outbox.listByCommandId(sharedCommandId), []);
const conversationOnly = yield* orchestrator.dispatch({
type: "checkpoint.rollback",
commandId: CommandId.make("runtime-rollback-conversation"),
threadId,
checkpointId,
scopeId: scope.id,
restoreFiles: false,
});
assert.deepEqual(
conversationOnly.storedEvents.map((stored) => stored.event.type),
["thread.metadata-updated", "checkpoint.rollback-requested"],
);
} else {
const error = yield* rollback.pipe(Effect.flip);
assert.instanceOf(error, Orchestrator.OrchestratorDispatchError);
assert.equal(
error.cause,
`Checkpoint ${checkpointId} is ${status} and cannot be restored.`,
);
assert.equal(yield* orchestrator.getThreadEventSequence(threadId), previousSequence);
assert.deepEqual(
yield* eventSink.readByCommandId({ commandId }).pipe(Stream.runCollect),
[],
);
assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
}
}
}).pipe(Effect.provide(Layer.fresh(layerTest))),
);
it.effect("resolves delivery intent against the active run and starts after it completes", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
const threadId = ThreadId.make("runtime-delivery-intent");
yield* orchestrator.dispatch({
type: "thread.create",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make("runtime-delivery-intent-create"),
threadId,
projectId: ProjectId.make("runtime-delivery-intent-project"),
title: "Delivery intent",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: process.cwd(),
});
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: CommandId.make("runtime-delivery-intent-first"),
threadId,
messageId: MessageId.make("runtime-delivery-intent-first"),
text: "Start work.",
attachments: [],
dispatchMode: { type: "start_immediately" },
});
const initial = yield* orchestrator.getThreadProjection(threadId);
const run = initial.runs[0]!;
const providerThread = initial.providerThreads[0]!;
const now = yield* DateTime.now;
const providerSession = {
id: providerThread.providerSessionId!,
driver,
providerInstanceId: modelSelection.instanceId,
status: "running" as const,
cwd: process.cwd(),
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
};
const providerTurn = {
id: ProviderTurnId.make("runtime-delivery-intent-turn"),
providerThreadId: providerThread.id,
nodeId: run.rootNodeId!,
runAttemptId: run.activeAttemptId,
nativeTurnRef: null,
ordinal: 1,
status: "running" as const,
startedAt: now,
completedAt: null,
};
yield* eventSink.write({
commandId: CommandId.make("runtime-delivery-intent-running"),
events: [
{
id: EventId.make("runtime-delivery-intent-run-event"),
type: "run.updated",
threadId,
runId: run.id,
occurredAt: now,
payload: { ...run, status: "running", startedAt: now },
},
{
id: EventId.make("runtime-delivery-intent-session-event"),
type: "provider-session.attached",
threadId,
occurredAt: now,
payload: providerSession,
},
{
id: EventId.make("runtime-delivery-intent-turn-event"),
type: "provider-turn.updated",
threadId,
runId: run.id,
occurredAt: now,
payload: providerTurn,
},
],
});
const sessionSpy = vi.spyOn(sessions, "get").mockReturnValue(
Effect.succeedSome({
providerSession,
} as ProviderAdapter.ProviderAdapterV2SessionRuntime),
);
yield* Effect.addFinalizer(() => Effect.sync(() => sessionSpy.mockRestore()));
const steerCommandId = CommandId.make("runtime-delivery-intent-auto");
const steerMessageId = MessageId.make("runtime-delivery-intent-auto");
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: steerCommandId,
threadId,
messageId: steerMessageId,
text: "Include this in the active work.",
attachments: [],
dispatchMode: { type: "start_immediately" },
deliveryIntent: "auto",
});
const steered = yield* orchestrator.getThreadProjection(threadId);
assert.lengthOf(steered.runs, 1);
assert.equal(
steered.messages.find((message) => message.id === steerMessageId)?.runId,
run.id,
);
assert.deepEqual(
(yield* outbox.listByCommandId(steerCommandId)).map((effect) => effect.request),
[
{
type: "provider-turn.steer",
providerSessionId: providerSession.id,
providerThreadId: providerThread.id,
providerTurnId: providerTurn.id,
messageId: steerMessageId,
},
],
);
yield* eventSink.write({
commandId: CommandId.make("runtime-delivery-intent-completed"),
events: [
{
id: EventId.make("runtime-delivery-intent-run-completed"),
type: "run.updated",
threadId,
runId: run.id,
occurredAt: now,
payload: { ...run, status: "completed", startedAt: now, completedAt: now },
},
{
id: EventId.make("runtime-delivery-intent-turn-completed"),
type: "provider-turn.updated",
threadId,
runId: run.id,
occurredAt: now,
payload: { ...providerTurn, status: "completed", completedAt: now },
},
],
});
const nextCommandId = CommandId.make("runtime-delivery-intent-next");
yield* orchestrator.dispatch({
type: "message.dispatch",
createdBy: "user",
creationSource: "web",
commandId: nextCommandId,
threadId,
messageId: MessageId.make("runtime-delivery-intent-next"),
text: "The previous run finished before this arrived.",
attachments: [],
dispatchMode: { type: "start_immediately" },
deliveryIntent: "restart",
});
const restarted = yield* orchestrator.getThreadProjection(threadId);
assert.deepEqual(
restarted.runs.map((candidate) => candidate.status),
["completed", "starting"],
);
assert.deepEqual(
(yield* outbox.listByCommandId(nextCommandId)).map((effect) => effect.request.type),
["provider-turn.start"],
);
}),
);
const seedPendingQuestion = Effect.fnUntraced(function* (
threadId: ThreadId,
runId: RunId | null,
responseCapability: OrchestrationV2RuntimeRequest["responseCapability"],
) {
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const requestId = RuntimeRequestId.make(`${threadId}:request`);
const nodeId = NodeId.make(`${threadId}:node`);
const itemId = TurnItemId.make(`${threadId}:item`);
yield* eventSink.write({
commandId: CommandId.make(`${threadId}:seed`),
events: [
{
id: EventId.make(`${threadId}:node-event`),
type: "node.updated",
threadId,
nodeId,
occurredAt: now,
payload: {
id: nodeId,
threadId,
runId,
parentNodeId: null,
rootNodeId: nodeId,
kind: "user_input_request",
status: "waiting",
countsForRun: false,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
runtimeRequestId: requestId,
checkpointScopeId: null,
startedAt: now,
completedAt: null,
},
},
{
id: EventId.make(`${threadId}:request-event`),
type: "runtime-request.updated",
threadId,
nodeId,
occurredAt: now,
payload: {
id: requestId,
nodeId,
providerTurnId: null,
nativeRequestRef: null,
kind: "user_input",
status: "pending",
responseCapability,
createdAt: now,
resolvedAt: null,
},
},
{
id: EventId.make(`${threadId}:item-event`),
type: "turn-item.updated",
threadId,
nodeId,
occurredAt: now,
payload: {
id: itemId,
type: "user_input_request",
threadId,
runId,
nodeId,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 0,
status: "waiting",
title: null,
startedAt: now,
completedAt: null,
updatedAt: now,
requestId,
questions: [
{ id: "color", header: "Color", question: "Which color?", options: [] },
{
id: "sizes",
header: "Sizes",
question: "Which sizes?",
options: [],
multiSelect: true,
},
{
id: "note",
header: "Note",
question: "Anything else?",
options: [],
required: false,
},
],
},
},
],
});
return { requestId, nodeId, itemId };
});
it.effect.each([
{ trigger: "startup", scope: "run" },
{ trigger: "shutdown", scope: "session" },
] as const)(
"recovers only reconstructable user questions and keeps their cards pending on $trigger ($scope)",
({ trigger, scope }) =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make(`runtime-recovered-question-${trigger}-${scope}`);
yield* orchestrator.dispatch({
type: "thread.create",
commandId: CommandId.make(`runtime-recovered-question-create-${trigger}-${scope}`),
createdBy: "user",
creationSource: "web",
threadId,
projectId: ProjectId.make(`runtime-recovered-question-project-${trigger}-${scope}`),
title: "Async question",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: process.cwd(),
});
if (scope === "run") {
yield* orchestrator.dispatch({
type: "message.dispatch",
commandId: CommandId.make(`${threadId}:run`),
threadId,
messageId: MessageId.make(`${threadId}:message`),
text: "Ask me a question",
attachments: [],
createdBy: "user",
creationSource: "web",
dispatchMode: { type: "start_immediately" },
});
}
const runId = (yield* orchestrator.getThreadProjection(threadId)).runs[0]?.id ?? null;
const { requestId, nodeId, itemId } = yield* seedPendingQuestion(threadId, runId, {
type: "live",
providerSessionId: ProviderSessionId.make(`lost-session-${trigger}-${scope}`),
});
const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
const original = yield* orchestrator.getThreadProjection(threadId);
const request = original.runtimeRequests[0]!;
const questionItem = original.turnItems.find((item) => item.type === "user_input_request")!;
const nativeThreadId = ThreadId.make(`runtime-recovered-native-child-${trigger}-${scope}`);
const nativeRequestId = RuntimeRequestId.make(
`runtime-recovered-native-request-${trigger}-${scope}`,
);
const approvalId = RuntimeRequestId.make(`runtime-recovered-approval-${trigger}-${scope}`);
const missingTextId = RuntimeRequestId.make(
`runtime-recovered-missing-text-${trigger}-${scope}`,
);
yield* eventSink.write({
events: [
{
id: EventId.make(`recovery-approval-${trigger}-${scope}`),
type: "runtime-request.updated",
threadId,
occurredAt: now,
payload: { ...request, id: approvalId, kind: "command" },
},
{
id: EventId.make(`recovery-missing-text-${trigger}-${scope}`),
type: "runtime-request.updated",
threadId,
occurredAt: now,
payload: { ...request, id: missingTextId },
},
{
id: EventId.make(`recovery-native-thread-${trigger}-${scope}`),
type: "thread.created",
threadId: nativeThreadId,
occurredAt: now,
payload: {
...original.thread,
id: nativeThreadId,
creationSource: "provider",
lineage: {
parentThreadId: threadId,
rootThreadId: threadId,
relationshipToParent: "subagent",
},
},
},
{
id: EventId.make(`recovery-native-request-${trigger}-${scope}`),
type: "runtime-request.updated",
threadId: nativeThreadId,
occurredAt: now,
payload: { ...request, id: nativeRequestId },
},
{
id: EventId.make(`recovery-native-item-${trigger}-${scope}`),
type: "turn-item.updated",
threadId: nativeThreadId,
occurredAt: now,
payload: {
...questionItem,
id: TurnItemId.make(`recovery-native-item-${trigger}-${scope}`),
type: "user_input_request",
threadId: nativeThreadId,
runId: null,
requestId: nativeRequestId,
questions: [
{ id: "color", header: "Color", question: "Which color?", options: [] },
],
},
},
],
});
yield* recovery.reconcile(trigger);
const recovered = yield* orchestrator.getThreadProjection(threadId);
if (scope === "run") {
assert.equal(recovered.runs[0]?.status, "cancelled");
} else {
assert.isEmpty(recovered.runs);
}
assert.deepEqual(
recovered.runtimeRequests.find((row) => row.id === requestId),
{
...request,
responseCapability: { type: "message" },
},
);
assert.equal(recovered.nodes.find((row) => row.id === nodeId)?.status, "waiting");
assert.equal(recovered.turnItems.find((row) => row.id === itemId)?.status, "waiting");
for (const id of [approvalId, missingTextId]) {
const closed = recovered.runtimeRequests.find((row) => row.id === id);
assert.equal(closed?.status, trigger === "startup" ? "expired" : "cancelled");
assert.equal(closed?.responseCapability.type, "not_resumable");
}
const native = yield* orchestrator.getThreadProjection(nativeThreadId);
assert.equal(
native.runtimeRequests[0]?.status,
trigger === "startup" ? "expired" : "cancelled",
);
assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable");
yield* recovery.reconcile("shutdown");
yield* recovery.reconcile("startup");
assert.deepEqual(
(yield* orchestrator.getThreadProjection(threadId)).runtimeRequests,
recovered.runtimeRequests,
);
yield* orchestrator.dispatch({
type: "runtime-request.respond",
commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`),
threadId,
requestId,
answers: { color: "Blue", sizes: ["Small", "Large"] },
});
const answered = yield* orchestrator.getThreadProjection(threadId);
assert.equal(
answered.runtimeRequests.find((row) => row.id === requestId)?.status,
"resolved",
);
assert.equal(answered.nodes.find((row) => row.id === nodeId)?.status, "completed");
assert.equal(answered.turnItems.find((row) => row.id === itemId)?.status, "completed");
assert.lengthOf(answered.runs, recovered.runs.length + 1);
assert.equal(answered.messages.at(-1)?.runId, answered.runs.at(-1)?.id);
assert.equal(
answered.messages.at(-1)?.text,
"Which color?\nBlue\n\nWhich sizes?\nSmall, Large",
);
assert.deepEqual(
(yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(
CommandId.make(`recovery-answer-${trigger}-${scope}`),
)).map((effect) => effect.request.type),
["provider-turn.start"],
);
}),
);
it.effect("answers an async question after its provider exits and commits the answer once", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
const eventSink = yield* EventSink.EventSinkV2;
const threadId = ThreadId.make("runtime-async-question");
yield* orchestrator.dispatch({
type: "thread.create",
commandId: CommandId.make("runtime-async-question-create"),
createdBy: "user",
creationSource: "web",
threadId,
projectId: ProjectId.make("runtime-async-question-project"),
title: "Async question",
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: process.cwd(),
});
const { requestId, nodeId, itemId } = yield* seedPendingQuestion(threadId, null, {
type: "message",
});
const outbox = yield* EffectOutbox.EffectOutboxV2;
const now = yield* DateTime.now;
const beforeSnooze = yield* orchestrator.getThreadProjection(threadId);
const snooze = {
type: "thread.snooze" as const,
commandId: CommandId.make("runtime-async-question-snooze"),
threadId,
snoozedUntil: "2099-07-25T09:00:00.000Z",
};
yield* orchestrator.dispatch(snooze);
const snoozed = yield* orchestrator.getThreadProjection(threadId);
assert.isNotNull(snoozed.thread.snoozedUntil);
assert.deepEqual(snoozed.runtimeRequests, beforeSnooze.runtimeRequests);
assert.deepEqual(snoozed.turnItems, beforeSnooze.turnItems);
assert.deepEqual(snoozed.nodes, beforeSnooze.nodes);
assert.deepEqual(snoozed.runs, beforeSnooze.runs);
// A question arriving after an earlier snooze must be acknowledged even
// when the user picks the same wake time again.
const priorSnooze = DateTime.makeUnsafe(DateTime.toEpochMillis(now) - 1_000);
yield* eventSink.write({
commandId: CommandId.make("runtime-async-question-earlier-snooze"),
events: [
{
id: EventId.make("runtime-async-question-earlier-snooze-event"),
type: "thread.snoozed",
threadId,
occurredAt: priorSnooze,
payload: { ...snoozed.thread, snoozedAt: priorSnooze },
},
],
});
yield* orchestrator.dispatch({
...snooze,
commandId: CommandId.make("runtime-async-question-resnooze"),
});
assert.deepEqual((yield* orchestrator.getThreadProjection(threadId)).thread.snoozedAt, now);
yield* orchestrator.dispatch({
type: "thread.unsnooze",
commandId: CommandId.make("runtime-async-question-wake"),
threadId,
reason: "user",
});
const awake = yield* orchestrator.getThreadProjection(threadId);
assert.isNull(awake.thread.snoozedUntil);
assert.deepEqual(awake.runtimeRequests, beforeSnooze.runtimeRequests);
const beforeInvalid = yield* orchestrator.getThreadProjection(threadId);
const beforeSequence = yield* eventSink.latestSequence();
for (const [index, answers] of [
{ color: " ", sizes: ["Small"] },
{ color: "Blue", sizes: [] },
{ color: "Blue", sizes: ["Small", " "] },
{ color: "Blue", sizes: ["Small"], extra: " " },
].entries()) {
const commandId = CommandId.make(`runtime-async-question-invalid:${index}`);
const invalid = yield* orchestrator
.dispatch({
type: "runtime-request.respond",
commandId,
threadId,
requestId,
answers,
})
.pipe(Effect.result);
assert.equal(invalid._tag, "Failure", JSON.stringify(answers));
assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), beforeInvalid);
assert.equal(yield* eventSink.latestSequence(), beforeSequence);
assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
}
const command = {