import { assert, it } from "@effect/vitest";
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
import * as NodeServices from "@effect/platform-node/NodeServices";
import {
CheckpointId,
CheckpointRef,
CheckpointScopeId,
MessageId,
GitCommandError,
type ModelSelection,
NodeId,
type OrchestrationV2AppThread,
type OrchestrationV2DomainEvent,
type OrchestrationV2ExecutionNode,
type OrchestrationV2RuntimeRequest,
type OrchestrationV2ProviderThread,
type OrchestrationV2Run,
type OrchestrationV2TurnItem,
ProviderDriverKind,
ProviderInstanceId,
ProviderThreadId,
ProviderTurnId,
RunAttemptId,
RunId,
RuntimeRequestId,
TurnItemId,
} from "@t3tools/contracts";
import * as Cause from "effect/Cause";
import * as DateTime from "effect/DateTime";
import * as Deferred from "effect/Deferred";
import * as Fiber from "effect/Fiber";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as SqlClient from "effect/sql/SqlClient";
import * as SqlError from "effect/sql/SqlError";
import * as Stream from "effect/Stream";
import { toolOutputImages } from "@t3tools/shared/toolOutput";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import * as ServerConfig from "../config.ts";
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
import * as EventSink from "./EventSink.ts";
import * as EventStore from "./EventStore.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
import {
makeProviderEventRoutingState,
type ProviderEventRouteIdentity,
routeProviderEvent,
selectInheritedBackgroundTurnItems,
} from "./RunExecutionService.ts";
const layerTestDatabase = SqlitePersistence.layerMemory;
const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe(
Layer.provide(layerTestDatabase),
);
const layerTestEventSink = EventSink.layer.pipe(
Layer.provide(Layer.mergeAll(layerTestStores, layerTestDatabase)),
);
const layerGit = GitVcsDriver.layer.pipe(
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "ingestor-git-" })),
Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
);
const layerEmptyInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({
getInstance: () => Effect.undefined,
});
const makeLayerTest = <E>(
ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>,
instances = layerEmptyInstances,
) =>
Layer.mergeAll(
instances,
layerTestDatabase,
layerTestStores,
layerTestEventSink,
IdAllocator.layer,
ThreadCommandExecutor.layer,
layerGit,
ProviderEventIngestor.layer.pipe(
Layer.provide(
Layer.mergeAll(
instances,
layerTestStores,
ingestorEventSink,
IdAllocator.layer,
ThreadCommandExecutor.layer,
layerGit,
),
),
),
);
const layerTest = makeLayerTest(layerTestEventSink);
// EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
const lockTimeout = new EventSink.EventSinkWriteError({
eventCount: 1,
cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
});
const constraintFailure = new EventSink.EventSinkWriteError({
eventCount: 1,
cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }),
});
/** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */
const layerFailingWrites = (
errors: ReadonlyArray<EventSink.EventSinkWriteError>,
matches: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => boolean = () => true,
) =>
makeLayerTest(
Layer.effect(
EventSink.EventSinkV2,
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const pending = [...errors];
const failNext = <A, R>(
events: ReadonlyArray<OrchestrationV2DomainEvent>,
write: Effect.Effect<A, EventSink.EventSinkV2Error, R>,
) => Effect.suspend(() => (matches(events) ? (pending.shift() ?? write) : write));
return EventSink.EventSinkV2.of({
...eventSink,
write: (input) => failNext(input.events, eventSink.write(input)),
writeIfRunCurrent: (input) => failNext(input.events, eventSink.writeIfRunCurrent(input)),
writeIfProviderThreadOwner: (input) =>
failNext(input.events, eventSink.writeIfProviderThreadOwner(input)),
});
}),
).pipe(Layer.provide(layerTestEventSink)),
);
const makeSharedWorktree = Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const git = yield* GitVcsDriver.GitVcsDriver;
const directory = yield* fs.makeTempDirectoryScoped({ prefix: "ingestor-worktree-" });
const repo = `${directory}/repo`;
const worktreePath = `${directory}/shared`;
yield* fs.makeDirectory(repo);
const command = (cwd: string, args: string[]) =>
git.execute({ operation: "ingestor-test", cwd, args });
yield* command(repo, ["init", "-b", "main"]);
yield* command(repo, [
"-c",
"user.name=Test",
"-c",
"user.email=test@example.test",
"commit",
"--allow-empty",
"-m",
"initial",
]);
yield* command(repo, ["worktree", "add", "-b", "t3/abcd1234", worktreePath]);
return { repo, worktreePath, command };
});
const modelSelection = {
instanceId: ProviderInstanceId.make("codex"),
model: "gpt-5.4",
} satisfies ModelSelection;
const CODEX_DRIVER = ProviderDriverKind.make("codex");
function threadCreatedEvent(
now: DateTime.Utc,
): Effect.Effect<
OrchestrationV2DomainEvent,
IdAllocator.IdAllocatorV2Error,
IdAllocator.IdAllocatorV2
> {
return Effect.gen(function* () {
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const projectId = yield* idAllocator.allocate.project({
fixtureName: "provider-event-ingestor",
});
const threadId = yield* idAllocator.allocate.thread({
fixtureName: "provider-event-ingestor",
projectId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-thread",
});
const thread: OrchestrationV2AppThread = {
createdBy: "user",
creationSource: "web",
id: threadId,
projectId,
title: "Provider event ingestor",
providerInstanceId: modelSelection.instanceId,
modelSelection: modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
branchPullRequest: null,
activeOrderKey: null,
activeProviderThreadId: providerThreadId,
lineage: {
parentThreadId: null,
relationshipToParent: null,
rootThreadId: threadId,
},
forkedFrom: null,
createdAt: now,
updatedAt: now,
archivedAt: null,
settledOverride: null,
settledAt: null,
lastVisitedAt: null,
deletedAt: null,
};
return {
id: yield* idAllocator.allocate.event({ threadId }),
type: "thread.created",
threadId,
occurredAt: now,
payload: thread,
};
});
}
const layer = it.layer(layerTest);
it.effect("refreshes the failing instance snapshot after a committed usage_limit terminal", () =>
Effect.scoped(
Effect.gen(function* () {
const refreshed = yield* Deferred.make<void>();
const adapter = yield* ProviderAdapter.ProviderAdapterV2.pipe(
Effect.provide(
Layer.mock(ProviderAdapter.ProviderAdapterV2)({
instanceId: ProviderInstanceId.make("cursor-work"),
driver: ProviderDriverKind.make("cursor"),
}),
),
);
const instance: ProviderInstance = {
instanceId: ProviderInstanceId.make("cursor-work"),
driverKind: ProviderDriverKind.make("cursor"),
continuationIdentity: {
driverKind: ProviderDriverKind.make("cursor"),
continuationKey: "work",
},
enabled: true,
displayName: undefined,
orchestrationAdapter: adapter,
textGeneration: {
generateBranchName: () => Effect.die("unused"),
generateCommitMessage: () => Effect.die("unused"),
generatePrContent: () => Effect.die("unused"),
generateThreadTitle: () => Effect.die("unused"),
generateReviewGuide: () => Effect.die("unused"),
generateTranscriptionPostProcessing: () => Effect.die("unused"),
},
snapshot: {
refreshUsage: Deferred.succeed(refreshed, undefined).pipe(Effect.asVoid),
getSnapshot: Effect.die("unused"),
refresh: Effect.die("unused"),
resolveMaintenance: () => Effect.die("unused"),
applyUsageLimits: () => Effect.die("unused"),
streamChanges: Stream.empty,
},
};
const lookups: ProviderInstanceId[] = [];
const instances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({
getInstance: (id) =>
Effect.sync(() => {
lookups.push(id);
return id === instance.instanceId ? instance : undefined;
}),
});
yield* Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
yield* eventSink.write({ events: [threadEvent] });
const input = {
providerSessionId: yield* idAllocator.allocate.providerSession({
providerInstanceId: instance.instanceId,
threadId: threadEvent.threadId,
}),
providerInstanceId: instance.instanceId,
threadId: threadEvent.threadId,
};
for (const failureClass of ["validation_error", "usage_limit"] as const) {
const stored = yield* ingestor.ingestNormalized({
...input,
event: {
type: "turn.terminal",
driver: instance.driverKind,
providerThreadId: idAllocator.derive.providerThread({
driver: instance.driverKind,
nativeThreadId: "limited",
}),
providerTurnId: idAllocator.derive.providerTurn({
driver: instance.driverKind,
nativeTurnId: failureClass,
}),
runOrdinal: 1,
failureItemOrdinal: 1,
status: "failed",
failure: makeProviderFailure({
class: failureClass,
message: "Provider rejected turn.",
}),
threadDisposition: "reusable",
},
});
assert.equal(stored.length, 1);
if (failureClass === "validation_error") assert.deepEqual(lookups, []);
}
assert.deepEqual(lookups, [instance.instanceId]);
yield* Deferred.await(refreshed);
}).pipe(Effect.provide(makeLayerTest(layerTestEventSink, instances)));
}),
),
);
it.effect("records accepted billed turn usage once without billing the context window", () => {
const recorded: Array<ProviderUsageAnalytics.ProviderTurnReport> = [];
const layerAnalytics = Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
runStarted: () => Effect.void,
turnCompleted: (report) =>
Effect.sync(() => {
recorded.push(report);
}),
runSettled: () => Effect.void,
});
return Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
yield* eventSink.write({ events: [threadEvent] });
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "billing-thread",
});
const providerTurn = {
id: idAllocator.derive.providerTurn({ driver: CODEX_DRIVER, nativeTurnId: "billing-turn" }),
providerThreadId,
nodeId: NodeId.make("node:billing-turn"),
runAttemptId: null,
nativeTurnRef: null,
ordinal: 1,
status: "completed" as const,
startedAt: now,
completedAt: DateTime.makeUnsafe(DateTime.toEpochMillis(now) + 120),
tokenUsage: {
inputTokens: 4000,
cachedInputTokens: 3000,
outputTokens: 100,
usedTokens: 4100,
updatedAt: DateTime.formatIso(now),
},
turnTokenUsage: {
usageStatus: "complete" as const,
usageScope: "main_agent" as const,
hasSubagents: false,
inputTokens: 40,
cachedInputTokens: 30,
outputTokens: 10,
},
};
const input = {
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
analyticsContext: {
modelSelection,
runtimeMode: "full-access" as const,
interactionMode: "default" as const,
origin: {
createdBy: "agent" as const,
creationSource: "provider" as const,
scheduled: false,
subagent: true,
},
},
event: { type: "provider_turn.updated" as const, driver: CODEX_DRIVER, providerTurn },
};
yield* ingestor.ingestNormalized(input);
yield* ingestor.ingestNormalized(input);
const ignored = yield* ingestor.ingestNormalized({
...input,
event: {
...input.event,
providerTurn: {
...providerTurn,
id: idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: "stale-billing-turn",
}),
},
},
writeIfRunCurrent: {
runId: RunId.make("missing-run"),
activeAttemptId: RunAttemptId.make("stale-attempt"),
expectedStatus: "running",
},
});
assert.isEmpty(ignored);
assert.lengthOf(recorded, 1);
assert.deepEqual(recorded[0], {
driver: CODEX_DRIVER,
providerTurn,
context: input.analyticsContext,
});
}).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics))));
});
layer("ProviderEventIngestorV2", (it) => {
it.effect("normalizes provider events through the real event log and projection store", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const eventStore = yield* EventStore.EventStoreV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const providerThread: OrchestrationV2ProviderThread = {
id: idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-thread",
}),
driver: CODEX_DRIVER,
providerInstanceId: modelSelection.instanceId,
providerSessionId,
appThreadId: threadEvent.threadId,
ownerNodeId: null,
nativeThreadRef: {
driver: CODEX_DRIVER,
nativeId: "native-thread",
strength: "strong",
},
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
};
yield* eventSink.write({ events: [threadEvent] });
const storedEvents = yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
event: {
type: "provider_thread.updated",
driver: CODEX_DRIVER,
providerThread,
},
});
const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
const storedDomainEvents = yield* eventStore.read({}).pipe(Stream.runCollect);
const afterFirstEvent = yield* eventStore
.read({ afterSequence: 1, threadId: threadEvent.threadId })
.pipe(Stream.runCollect);
const latestThreadSequence = yield* eventStore.latestSequence({
threadId: threadEvent.threadId,
});
assert.equal(storedEvents.length, 1);
assert.equal(storedEvents[0]?.event.type, "provider-thread.updated");
assert.deepEqual(
projection.providerThreads.map((thread) => thread.id),
[providerThread.id],
);
assert.deepEqual(
Array.from(storedDomainEvents).map((stored) => stored.event.type),
["thread.created", "provider-thread.updated"],
);
assert.deepEqual(
Array.from(storedDomainEvents).map((stored) => stored.sequence),
[1, 2],
);
assert.deepEqual(
Array.from(afterFirstEvent).map((stored) => stored.event.type),
["provider-thread.updated"],
);
assert.equal(latestThreadSequence, 2);
}),
);
it.effect("skips a node update only when it matches the stored node", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const threadId = threadEvent.threadId;
yield* eventSink.write({ events: [threadEvent] });
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
const nodeId = NodeId.make(`${threadId}:approval`);
const node: OrchestrationV2ExecutionNode = {
id: nodeId,
threadId,
runId: null,
parentNodeId: null,
rootNodeId: nodeId,
kind: "approval_request",
status: "waiting",
countsForRun: false,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
runtimeRequestId: null,
checkpointScopeId: null,
startedAt: now,
completedAt: null,
};
const ingestNode = (sent: OrchestrationV2ExecutionNode) =>
ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId,
event: { type: "node.updated", driver: CODEX_DRIVER, node: sent },
});
yield* eventSink.withProviderBatch(
Effect.gen(function* () {
assert.lengthOf(yield* ingestNode(node), 1);
assert.isEmpty(yield* ingestNode({ ...node }));
}),
);
yield* eventSink.write({
events: [
{
id: yield* idAllocator.allocate.event({ threadId }),
type: "node.updated",
threadId,
occurredAt: now,
payload: { ...node, status: "completed", completedAt: now },
},
],
});
assert.lengthOf(yield* ingestNode(node), 1);
const projection = yield* projectionStore.getThreadProjection(threadId);
assert.equal(projection.nodes.find((entry) => entry.id === nodeId)?.status, "waiting");
}),
);
it.effect("rolls back runs a provider rewound off its active branch", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const threadId = threadEvent.threadId;
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
const rewoundThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "rewound-thread",
});
const otherThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "other-thread",
});
const scopeId = CheckpointScopeId.make("checkpoint-scope:rewound");
const seedRun = Effect.fnUntraced(function* (input: {
readonly ordinal: number;
readonly providerThreadId: ProviderThreadId;
readonly status: "completed" | "interrupted" | "running";
readonly nativeTurnId: string | null;
}) {
const runId = RunId.make(`run:rewound:${input.ordinal}`);
const attemptId = RunAttemptId.make(`attempt:rewound:${input.ordinal}`);
const nodeId = NodeId.make(`node:rewound:${input.ordinal}`);
const providerTurnId = ProviderTurnId.make(`provider-turn:rewound:${input.ordinal}`);
const completedAt = input.status === "running" ? null : now;
const eventId = () => idAllocator.allocate.event({ threadId });
const events: Array<OrchestrationV2DomainEvent> = [
{
id: yield* eventId(),
type: "run.created",
threadId,
occurredAt: now,
payload: {
id: runId,
threadId,
ordinal: input.ordinal,
providerInstanceId: modelSelection.instanceId,
modelSelection,
providerThreadId: input.providerThreadId,
userMessageId: MessageId.make(`message:rewound:${input.ordinal}`),
rootNodeId: nodeId,
activeAttemptId: attemptId,
status: input.status,
requestedAt: now,
startedAt: now,
completedAt,
checkpointId: null,
contextHandoffId: null,
},
},
{
id: yield* eventId(),
type: "run-attempt.created",
threadId,
occurredAt: now,
payload: {
id: attemptId,
runId,
attemptOrdinal: 1,
rootNodeId: nodeId,
providerInstanceId: modelSelection.instanceId,
providerThreadId: input.providerThreadId,
providerTurnId,
reason: "initial",
status: input.status,
startedAt: now,
completedAt,
},
},
{
id: yield* eventId(),
type: "provider-turn.updated",
threadId,
occurredAt: now,
payload: {
id: providerTurnId,
providerThreadId: input.providerThreadId,
nodeId,
runAttemptId: attemptId,
nativeTurnRef:
input.nativeTurnId === null
? {
driver: CODEX_DRIVER,
nativeId: `synthetic:${input.ordinal}`,
strength: "weak",
}
: { driver: CODEX_DRIVER, nativeId: input.nativeTurnId, strength: "strong" },
ordinal: input.ordinal,
status: input.status,
startedAt: now,
completedAt,
},
},
{
id: yield* eventId(),
type: "node.updated",
threadId,
occurredAt: now,
payload: {
id: nodeId,
threadId,
runId,
parentNodeId: null,
rootNodeId: nodeId,
kind: "root_turn",
status: input.status,
countsForRun: true,
providerThreadId: input.providerThreadId,
providerTurnId,
nativeItemRef: null,
runtimeRequestId: null,
checkpointScopeId: scopeId,
startedAt: now,
completedAt,
},
},
...(input.status === "completed"
? [
{
id: yield* eventId(),
type: "checkpoint.captured" as const,
threadId,
occurredAt: now,
payload: {
id: CheckpointId.make(`checkpoint:rewound:${input.ordinal}`),
threadId,
scopeId,
runId,
nodeId,
parentCheckpointId: null,
ordinalWithinScope: input.ordinal,
appRunOrdinal: input.ordinal,
ref: CheckpointRef.make(`refs/t3/checkpoints/rewound/${input.ordinal}`),
status: "ready" as const,
files: [],
capturedAt: now,
},
},
]
: []),
];
yield* eventSink.write({ events });
return runId;
});
yield* eventSink.write({ events: [threadEvent] });
const otherProviderRun = yield* seedRun({
ordinal: 1,
providerThreadId: otherThreadId,
status: "completed",
nativeTurnId: "other-turn",
});
const keptRun = yield* seedRun({
ordinal: 2,
providerThreadId: rewoundThreadId,
status: "completed",
nativeTurnId: "kept-turn",
});
const abandonedRun = yield* seedRun({
ordinal: 3,
providerThreadId: rewoundThreadId,
status: "completed",
nativeTurnId: "abandoned-turn",
});
const unlocatedRun = yield* seedRun({
ordinal: 4,
providerThreadId: rewoundThreadId,
status: "interrupted",
nativeTurnId: null,
});
const rewindingRun = yield* seedRun({
ordinal: 5,
providerThreadId: rewoundThreadId,
status: "running",
nativeTurnId: null,
});
yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId,
runId: rewindingRun,
event: {
type: "provider_thread.updated",
driver: CODEX_DRIVER,
providerThread: {
id: rewoundThreadId,
driver: CODEX_DRIVER,
providerInstanceId: modelSelection.instanceId,
providerSessionId,
appThreadId: threadId,
ownerNodeId: null,
nativeThreadRef: {
driver: CODEX_DRIVER,
nativeId: "rewound-thread",
strength: "strong",
},
nativeConversationHeadRef: {
driver: CODEX_DRIVER,
nativeId: "kept-turn-reply",
strength: "strong",
},
status: "idle",
firstRunOrdinal: 2,
lastRunOrdinal: 5,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
},
retainedNativeTurnIds: ["turn-started-outside-t3", "kept-turn"],
},
});
const projection = yield* projectionStore.getThreadProjection(threadId);
const runIds = [otherProviderRun, keptRun, abandonedRun, unlocatedRun, rewindingRun];
assert.deepEqual(
runIds.map((runId) => projection.runs.find((run) => run.id === runId)?.status),
["completed", "completed", "rolled_back", "rolled_back", "running"],
);
assert.deepEqual(
runIds.map((runId) => projection.nodes.find((node) => node.runId === runId)?.status),
["completed", "completed", "rolled_back", "rolled_back", "running"],
);
assert.deepEqual(
[otherProviderRun, keptRun, abandonedRun].map(
(runId) =>
projection.checkpoints.find((checkpoint) => checkpoint.runId === runId)?.status,
),
["ready", "ready", "stale"],
);
assert.equal(
projection.providerThreads.find((thread) => thread.id === rewoundThreadId)?.lastRunOrdinal,
5,
);
}),
);
it.effect(
"treats successful provider terminal markers as non-persisted orchestration control signals",
() =>
Effect.gen(function* () {
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const projectId = yield* idAllocator.allocate.project({
fixtureName: "provider-event-terminal",
});
const threadId = yield* idAllocator.allocate.thread({
fixtureName: "provider-event-terminal",
projectId,
});
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
const normalized = yield* ingestor.normalize({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId,
event: {
type: "turn.terminal",
driver: CODEX_DRIVER,
providerThreadId: idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-thread",
}),
providerTurnId: idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: "native-turn",
}),
runOrdinal: 1,
status: "completed",
failure: null,
threadDisposition: "reusable",
},
});
assert.deepEqual(normalized, []);
}),
);
it.effect("persists an interrupted run's inherited terminal through the live run router", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const priorRunId = RunId.make("run:provider-event-inherited:prior");
const currentRunId = RunId.make("run:provider-event-inherited:current");
const itemId = TurnItemId.make("turn-item:provider-event-inherited");
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-thread-inherited",
});
const providerTurnId = idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: "native-turn-inherited",
});
const runningItem = {
id: itemId,
threadId: threadEvent.threadId,
runId: priorRunId,
nodeId: NodeId.make("node:provider-event-inherited"),
providerThreadId,
providerTurnId,
nativeItemRef: null,
parentItemId: null,
ordinal: 101,
status: "running",
title: "Inherited background command",
startedAt: now,
completedAt: null,
updatedAt: now,
type: "command_execution",
input: "sleep 60",
} satisfies OrchestrationV2TurnItem;
const terminalItem = {
...runningItem,
status: "completed" as const,
completedAt: now,
updatedAt: now,
};
yield* eventSink.write({ events: [threadEvent] });
yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
runId: priorRunId,
event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem },
});
const identity: ProviderEventRouteIdentity = {
threadId: threadEvent.threadId,
runId: currentRunId,
attemptId: RunAttemptId.make("attempt:provider-event-inherited:current"),
providerThreadId,
};
const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({
threadId: threadEvent.threadId,
currentProviderThreadId: providerThreadId,
currentRunOrdinal: 2,
runs: [
{
id: priorRunId,
threadId: threadEvent.threadId,
ordinal: 1,
status: "interrupted",
} as OrchestrationV2Run,
{
id: currentRunId,
threadId: threadEvent.threadId,
ordinal: 2,
status: "running",
} as OrchestrationV2Run,
],
turnItems: [runningItem],
});
const routeState = makeProviderEventRoutingState({
identity,
inheritedBackgroundTurnItems,
providerTurnId: null,
});
const terminalEvent = {
type: "turn_item.updated",
driver: CODEX_DRIVER,
turnItem: terminalItem,
} as const;
const [accepted] = routeProviderEvent(terminalEvent, identity, routeState);
assert.isTrue(accepted);
const stored = yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
runId: currentRunId,
event: terminalEvent,
});
const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
const persisted = projection.turnItems.find((item) => item.id === itemId);
assert.equal(stored.length, 1);
assert.equal(stored[0]?.event.type, "turn-item.updated");
assert.equal(persisted?.runId, priorRunId);
assert.equal(persisted?.threadId, threadEvent.threadId);
assert.equal(persisted?.status, "completed");
}),
);
it.effect("persists a completed run's late background terminal exactly once", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const eventStore = yield* EventStore.EventStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const priorRunId = RunId.make("run:provider-event-completed:prior");
const currentRunId = RunId.make("run:provider-event-completed:current");
const itemId = TurnItemId.make("turn-item:provider-event-completed");
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-thread-completed",
});
const providerTurnId = idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: "native-turn-completed",
});
const runningItem = {
id: itemId,
threadId: threadEvent.threadId,
runId: priorRunId,
nodeId: NodeId.make("node:provider-event-completed"),
providerThreadId,
providerTurnId,
nativeItemRef: null,
parentItemId: null,
ordinal: 101,
status: "running",
title: "Completed run background command",
startedAt: now,
completedAt: null,
updatedAt: now,
type: "command_execution",
input: "sleep 60",
} satisfies OrchestrationV2TurnItem;
const terminalEvent = {
type: "turn_item.updated",
driver: CODEX_DRIVER,
turnItem: {
...runningItem,
status: "completed" as const,
completedAt: now,
updatedAt: now,
},
} as const;
yield* eventSink.write({ events: [threadEvent] });
yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
runId: priorRunId,
event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem },
});
const priorIdentity: ProviderEventRouteIdentity = {
threadId: threadEvent.threadId,
runId: priorRunId,
attemptId: RunAttemptId.make("attempt:provider-event-completed:prior"),
providerThreadId,
};
const currentIdentity: ProviderEventRouteIdentity = {
threadId: threadEvent.threadId,
runId: currentRunId,
attemptId: RunAttemptId.make("attempt:provider-event-completed:current"),
providerThreadId,
};
const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({
threadId: threadEvent.threadId,
currentProviderThreadId: providerThreadId,
currentRunOrdinal: 2,
runs: [
{
id: priorRunId,
threadId: threadEvent.threadId,
ordinal: 1,
status: "completed",
} as OrchestrationV2Run,
{
id: currentRunId,
threadId: threadEvent.threadId,
ordinal: 2,
status: "running",
} as OrchestrationV2Run,
],
turnItems: [runningItem],
});
const routers = [
{
identity: priorIdentity,
state: makeProviderEventRoutingState({
identity: priorIdentity,
providerTurnId: providerTurnId,
}),
},
{
identity: currentIdentity,
state: makeProviderEventRoutingState({
identity: currentIdentity,
inheritedBackgroundTurnItems,
providerTurnId: null,
}),
},
];
const acceptedRouters = routers.filter(
({ identity, state }) => routeProviderEvent(terminalEvent, identity, state)[0],
);
yield* Effect.forEach(
acceptedRouters,
({ identity }) =>
ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
runId: identity.runId,
event: terminalEvent,
}),
{ concurrency: 1 },
);
const storedEvents = yield* eventStore
.read({ threadId: threadEvent.threadId })
.pipe(Stream.runCollect);
const storedTerminals = Array.from(storedEvents).filter(
(stored) =>
stored.event.type === "turn-item.updated" &&
stored.event.payload.id === itemId &&
stored.event.payload.status === "completed",
);
assert.equal(storedTerminals.length, 1);
assert.equal(acceptedRouters.length, 1);
assert.equal(acceptedRouters[0]?.identity.runId, priorRunId);
}),
);
it.effect.each(["interrupted", "control"] as const)(
"dismisses only native questions when a provider turn ends with %s",
(terminal) =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const eventStore = yield* EventStore.EventStoreV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const threadId = threadEvent.threadId;
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: `${threadId}:questions`,
});
const providerTurnId = idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: `${threadId}:questions`,
});
const otherTurnId = idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: `${threadId}:other-turn`,
});
const specs = [
{ key: "native", responseCapability: { type: "live", providerSessionId } },
{
key: "unavailable",
responseCapability: { type: "not_resumable", reason: "Turn ended" },
},
{ key: "message", responseCapability: { type: "message" } },
{ key: "answered", responseCapability: { type: "live", providerSessionId } },
{ key: "other-turn", responseCapability: { type: "live", providerSessionId } },
{ key: "approval", responseCapability: { type: "live", providerSessionId } },
] as const;
const fixtures = specs.map((spec, ordinal) => {
const nodeId = NodeId.make(`${threadId}:${spec.key}`);
const resolved = spec.key === "answered";
const request: OrchestrationV2RuntimeRequest = {
id: RuntimeRequestId.make(`${threadId}:${spec.key}`),
nodeId,
providerTurnId: spec.key === "other-turn" ? otherTurnId : providerTurnId,
nativeRequestRef: null,
kind: spec.key === "approval" ? "command" : "user_input",
status: resolved ? "resolved" : "pending",
responseCapability: spec.responseCapability,
createdAt: now,
resolvedAt: resolved ? now : null,
};
const node: OrchestrationV2ExecutionNode = {
id: nodeId,
threadId,
runId: null,
parentNodeId: null,
rootNodeId: nodeId,
kind: spec.key === "approval" ? "approval_request" : "user_input_request",
status: resolved ? "completed" : "waiting",
countsForRun: false,
providerThreadId,
providerTurnId: request.providerTurnId,
nativeItemRef: null,
runtimeRequestId: request.id,
checkpointScopeId: null,
startedAt: now,
completedAt: resolved ? now : null,
};
const item: OrchestrationV2TurnItem = {
id: TurnItemId.make(`${threadId}:${spec.key}`),
threadId,
runId: null,
nodeId,
providerThreadId,
providerTurnId: request.providerTurnId,
nativeItemRef: null,
parentItemId: null,
ordinal,
status: resolved ? "completed" : "waiting",
title: "Which option?",
startedAt: now,
completedAt: resolved ? now : null,
updatedAt: now,
...(spec.key === "approval"
? { type: "approval_request", requestId: request.id, requestKind: "command" }
: {
type: "user_input_request",
requestId: request.id,
questions: [],
...(spec.key === "message" ? { responseMode: "message" as const } : {}),
}),
};
return { key: spec.key, request, node, item };
});
const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent];
for (const fixture of fixtures) {
for (const payload of [
{ type: "runtime-request.updated" as const, payload: fixture.request },
{ type: "node.updated" as const, payload: fixture.node },
{ type: "turn-item.updated" as const, payload: fixture.item },
]) {
seedEvents.push({
id: yield* idAllocator.allocate.event({ threadId }),
threadId,
occurredAt: now,
...payload,
});
}
}
yield* eventSink.write({ events: seedEvents });
const input = {
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId,
event:
terminal === "control"
? {
type: "turn.terminal" as const,
driver: CODEX_DRIVER,
providerThreadId,
providerTurnId,
runOrdinal: 1,
status: "completed" as const,
failure: null,
threadDisposition: "reusable" as const,
}
: {
type: "provider_turn.updated" as const,
driver: CODEX_DRIVER,
providerTurn: {
id: providerTurnId,
providerThreadId,
nodeId: NodeId.make(`${threadId}:root`),
runAttemptId: null,
nativeTurnRef: null,
ordinal: 1,
status: terminal,
startedAt: now,
completedAt: now,
},
},
};
const stored = yield* ingestor.ingestNormalized(input);
const projection = yield* projectionStore.getThreadProjection(threadId);
for (const fixture of fixtures) {
const closed = fixture.key === "native" || fixture.key === "unavailable";
const request = projection.runtimeRequests.find(
(item) => item.id === fixture.request.id,
)!;
const node = projection.nodes.find((item) => item.id === fixture.node.id)!;
const item = projection.turnItems.find((item) => item.id === fixture.item.id)!;
if (closed) {
assert.equal(request.status, "cancelled");
assert.isNotNull(request.resolvedAt);
assert.equal(node.status, "cancelled");
assert.isNotNull(node.completedAt);
assert.equal(item.status, "cancelled");
assert.isNotNull(item.completedAt);
} else {
assert.equal(request.status, fixture.request.status);
assert.equal(node.status, fixture.node.status);
assert.equal(item.status, fixture.item.status);
}
}
assert.equal(
stored.filter((entry) => entry.event.type === "runtime-request.updated").length,
2,
);
assert.isEmpty(
(yield* projectionStore.getPendingNativeUserInputs(threadId, providerTurnId))
.runtimeRequests,
);
const replayed = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
assert.equal(
replayed.filter(
(entry) =>
entry.event.type === "runtime-request.updated" &&
entry.event.payload.status === "cancelled",
).length,
2,
);
const repeated = yield* ingestor.ingestNormalized(input);
assert.isFalse(repeated.some((entry) => entry.event.type === "runtime-request.updated"));
}),
);
it.effect(
"preserves an answer committed after terminal normalization reads a pending question",
() =>
Effect.scoped(
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const eventStore = yield* EventStore.EventStoreV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const now = yield* DateTime.now;
const threadEvent = yield* threadCreatedEvent(now);
const threadId = threadEvent.threadId;
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: `${threadId}:race`,
});
const providerTurnId = idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: `${threadId}:race`,
});
const nodeId = NodeId.make(`${threadId}:question`);
const request: OrchestrationV2RuntimeRequest = {
id: RuntimeRequestId.make(`${threadId}:question`),
nodeId,
providerTurnId,
nativeRequestRef: null,
kind: "user_input",
status: "pending",
responseCapability: { type: "live", providerSessionId },
createdAt: now,
resolvedAt: null,
};
const node: OrchestrationV2ExecutionNode = {
id: nodeId,
threadId,
runId: null,
parentNodeId: null,
rootNodeId: nodeId,
kind: "user_input_request",
status: "waiting",
countsForRun: false,
providerThreadId,
providerTurnId,
nativeItemRef: null,
runtimeRequestId: request.id,
checkpointScopeId: null,
startedAt: now,
completedAt: null,
};
const item: OrchestrationV2TurnItem = {
id: TurnItemId.make(`${threadId}:question`),
threadId,
runId: null,
nodeId,
providerThreadId,
providerTurnId,
nativeItemRef: null,
parentItemId: null,
ordinal: 1,
status: "waiting",
title: "Which option?",
startedAt: now,
completedAt: null,
updatedAt: now,
type: "user_input_request",
requestId: request.id,
questions: [],
};
const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent];
for (const payload of [
{ type: "runtime-request.updated" as const, payload: request },
{ type: "node.updated" as const, payload: node },
{ type: "turn-item.updated" as const, payload: item },
])
seedEvents.push({
id: yield* idAllocator.allocate.event({ threadId }),
threadId,
occurredAt: now,
...payload,
});
yield* eventSink.write({ events: seedEvents });
const normalized = yield* Deferred.make<void>();
const releaseTerminalWrite = yield* Deferred.make<void>();
const gatedSink = EventSink.EventSinkV2.of({
...eventSink,
write: (input) =>
Deferred.succeed(normalized, undefined).pipe(
Effect.andThen(Deferred.await(releaseTerminalWrite)),
Effect.andThen(eventSink.write(input)),
),
});
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe(
Effect.provide(Layer.fresh(ProviderEventIngestor.layer)),
Effect.provideService(EventSink.EventSinkV2, gatedSink),
);
const terminal = yield* ingestor
.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId,
event: {
type: "provider_turn.updated",
driver: CODEX_DRIVER,
providerTurn: {
id: providerTurnId,
providerThreadId,
nodeId,
runAttemptId: null,
nativeTurnRef: null,
ordinal: 1,
status: "completed",
startedAt: now,
completedAt: now,
},
},
})
.pipe(Effect.forkScoped);
yield* Deferred.await(normalized);
const answers = { decision: "Use the existing workspace" };
const responseEvents: Array<OrchestrationV2DomainEvent> = [];
for (const payload of [
{
type: "runtime-request.updated" as const,
payload: { ...request, status: "resolved" as const, answers, resolvedAt: now },
},
{
type: "node.updated" as const,
payload: { ...node, status: "completed" as const, completedAt: now },
},
{
type: "turn-item.updated" as const,
payload: { ...item, status: "completed" as const, completedAt: now },
},
])
responseEvents.push({
id: yield* idAllocator.allocate.event({ threadId }),
threadId,
occurredAt: now,
...payload,
});
yield* eventSink.write({ events: responseEvents });
yield* Deferred.succeed(releaseTerminalWrite, undefined);
const committedTerminal = yield* Fiber.join(terminal);
assert.deepEqual(
committedTerminal.map((stored) => stored.event.type),
["provider-turn.updated"],
);
const projection = yield* projections.getThreadProjection(threadId);
const answered = projection.runtimeRequests.find((entry) => entry.id === request.id)!;
assert.equal(answered.status, "resolved");
assert.deepEqual(answered.answers, answers);
assert.equal(projection.nodes.find((entry) => entry.id === nodeId)!.status, "completed");
assert.equal(
projection.turnItems.find((entry) => entry.id === item.id)!.status,
"completed",
);
const history = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
assert.isFalse(
history.some(
(stored) =>
stored.event.type === "runtime-request.updated" &&
stored.event.payload.status === "cancelled",
),
);
}),
),
);
it.effect("persists a failed provider terminal as one expected error item", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const retryStartedAt = DateTime.makeUnsafe(DateTime.toEpochMillis(now) - 5_000);
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const providerThreadId = idAllocator.derive.providerThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-thread-failed",
});
const providerTurnId = idAllocator.derive.providerTurn({
driver: CODEX_DRIVER,
nativeTurnId: "native-turn-failed",
});
yield* eventSink.write({ events: [threadEvent] });
const stored = yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
event: {
type: "turn.terminal",
driver: CODEX_DRIVER,
providerThreadId,
providerTurnId,
runOrdinal: 1,
failureItemOrdinal: 102,
status: "failed",
failure: makeProviderFailure({
message: "Invalid reasoning effort.",
code: "invalid_request",
class: "validation_error",
}),
retry: {
attempt: 3,
maxAttempts: 3,
retryDelayMs: 2_000,
},
retryStartedAt,
threadDisposition: "reusable",
},
});
const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
const errorItems = projection.visibleTurnItems.filter(
(candidate) => candidate.item.type === "error",
);
assert.equal(stored.length, 1);
assert.equal(stored[0]?.event.type, "turn-item.updated");
assert.equal(errorItems.length, 1);
const errorItem = errorItems[0]?.item;
assert.equal(errorItem?.type, "error");
if (errorItem?.type !== "error") return;
assert.equal(errorItem.failure.message, "Invalid reasoning effort.");
assert.equal(errorItem.failure.code, "invalid_request");
assert.deepEqual(errorItem.retry, {
attempt: 3,
maxAttempts: 3,
retryDelayMs: 2_000,
});
const errorStartedAt = errorItem.startedAt;
assert.ok(errorStartedAt);
assert.equal(DateTime.toEpochMillis(errorStartedAt), DateTime.toEpochMillis(retryStartedAt));
assert.equal(errorItem.providerThreadId, providerThreadId);
assert.equal(errorItem.providerTurnId, providerTurnId);
}),
);
it.effect("stores tool image bytes only where a tool-output-image asset serves them", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const eventStore = yield* EventStore.EventStoreV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const readBase64 = Buffer.alloc(30_000, 7).toString("base64");
const screenshotBase64 = Buffer.alloc(20_000, 9).toString("base64");
const toolItem = (
id: string,
ordinal: number,
toolName: string,
output: unknown,
): OrchestrationV2TurnItem => ({
id: TurnItemId.make(id),
threadId: threadEvent.threadId,
runId: null,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal,
status: "completed",
title: toolName,
startedAt: now,
completedAt: now,
updatedAt: now,
type: "dynamic_tool",
toolName,
input: {},
output,
});
const read = toolItem("turn-item:read-image", 1, "Read", {
type: "image",
file: { base64: readBase64, type: "image/png", originalSize: 30_000 },
});
const screenshot = toolItem("turn-item:screenshot", 2, "mcp__t3-code__device_screenshot", {
content: [
{
type: "image",
source: { type: "base64", media_type: "image/png", data: screenshotBase64 },
},
],
});
yield* eventSink.write({ events: [threadEvent] });
for (const turnItem of [read, screenshot]) {
yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem },
});
}
const storedEvents = yield* eventStore
.read({ threadId: threadEvent.threadId, eventType: "turn-item.updated" })
.pipe(Stream.runCollect);
const storedJson = JSON.stringify(Array.from(storedEvents, (stored) => stored.event));
const projectedRead = yield* projectionStore.getTurnItem({
threadId: threadEvent.threadId,
itemId: read.id,
});
const projectedScreenshot = yield* projectionStore.getTurnItem({
threadId: threadEvent.threadId,
itemId: screenshot.id,
});
assert.equal(storedJson.includes(readBase64), false);
assert.equal(storedJson.includes(screenshotBase64), true);
assert.deepEqual(projectedRead?.type === "dynamic_tool" ? projectedRead.output : null, {
type: "image",
file: { type: "image/png", originalSize: 30_000, sizeBytes: 30_000 },
});
assert.deepEqual(
toolOutputImages(
projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null,
),
[{ mimeType: "image/png", data: screenshotBase64 }],
);
}),
);
it.effect("stores a bounded copy of a large file-change diff in both SQLite tables", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const threadEvent = yield* threadCreatedEvent(now);
const providerSessionId = yield* idAllocator.allocate.providerSession({
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
});
const largeDiff = `@@ -0,0 +1 @@\n${"+generated line\n".repeat(150_000)}`;
const fileChange = (
id: string,
ordinal: number,
diffStr: string,
): OrchestrationV2TurnItem => ({
id: TurnItemId.make(id),
threadId: threadEvent.threadId,
runId: null,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal,
status: "completed",
title: null,
startedAt: now,
completedAt: now,
updatedAt: now,
type: "file_change",
fileName: "src/generated.ts",
diffStr,
});
const large = fileChange("turn-item:large-diff", 1, largeDiff);
const small = fileChange("turn-item:small-diff", 2, "@@ -1 +1 @@\n-old\n+new\n");
yield* eventSink.write({ events: [threadEvent] });
for (const turnItem of [large, small]) {
yield* ingestor.ingestNormalized({
providerSessionId,
providerInstanceId: modelSelection.instanceId,
threadId: threadEvent.threadId,
event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem },
});
}
const sql = yield* SqlClient.SqlClient;
const storedBytes = yield* sql<{ readonly events: number; readonly projection: number }>`
SELECT
(SELECT SUM(LENGTH(payload_json)) FROM orchestration_events
WHERE stream_id = ${threadEvent.threadId} AND event_type = 'turn-item.updated') AS events,
(SELECT SUM(LENGTH(payload_json)) FROM orchestration_v2_projection_turn_items
WHERE thread_id = ${threadEvent.threadId}) AS projection
`;
const projectedLarge = yield* projectionStore.getTurnItem({
threadId: threadEvent.threadId,
itemId: large.id,
});
const projectedSmall = yield* projectionStore.getTurnItem({
threadId: threadEvent.threadId,
itemId: small.id,
});
assert.isAbove(largeDiff.length, 2_000_000);
assert.isBelow(storedBytes[0]!.events, 100_000);
assert.isBelow(storedBytes[0]!.projection, 100_000);
assert.isTrue(projectedLarge?.type === "file_change");
if (projectedLarge?.type !== "file_change") return;
assert.equal(projectedLarge.fileName, "src/generated.ts");
assert.isBelow(projectedLarge.diffStr?.length ?? 0, 40_000);
assert.isTrue(largeDiff.startsWith(projectedLarge.diffStr?.split("\n…")[0] ?? "-"));
assert.deepEqual(projectedSmall, small);
}),
);
it.effect("routes provider-owned child artifacts to their child app thread", () =>
Effect.gen(function* () {
const now = yield* DateTime.now;
const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
const idAllocator = yield* IdAllocator.IdAllocatorV2;
const rootEvent = yield* threadCreatedEvent(now);
if (rootEvent.type !== "thread.created") {
throw new Error("Expected a thread.created fixture event");
}
const childThreadId = idAllocator.derive.threadFromProviderThread({
driver: CODEX_DRIVER,
nativeThreadId: "native-subagent-thread",
});
const childRootNodeId = NodeId.make("node:subagent-root");
const childThread: OrchestrationV2AppThread = {
...rootEvent.payload,
id: childThreadId,
title: "inspect package",
activeProviderThreadId: null,
lineage: {