import * as NodeSqlite from "node:sqlite";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
import {
CheckpointId,
CheckpointRef,
CheckpointScopeId,
CommandId,
ContextTransferId,
EventId,
MessageId,
type ModelSelection,
NodeId,
type OrchestrationV2AppThread,
type OrchestrationV2DomainEvent,
type OrchestrationV2Run,
type OrchestrationV2ThreadProjection,
type OrchestrationV2TurnItem,
ProjectId,
ProviderDriverKind,
ProviderInstanceId,
ProviderSessionId,
ProviderThreadId,
RunAttemptId,
RunId,
ThreadId,
TurnItemId,
} from "@t3tools/contracts";
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import * as ConfigProvider from "effect/ConfigProvider";
import * as Queue from "effect/Queue";
import * as Arr from "effect/Array";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Exit from "effect/Exit";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Ref from "effect/Ref";
import * as Scheduler from "effect/Scheduler";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import * as Tracer from "effect/Tracer";
import * as SqlClient from "effect/sql/SqlClient";
import * as Statement from "effect/sql/Statement";
import { vi } from "vite-plus/test";
import * as ServerConfig from "../config.ts";
import * as DatabaseStorage from "../persistence/DatabaseStorage.ts";
import * as ServerSettings from "../serverSettings.ts";
import { LIVE_STREAM_MAX_ITEMS, LiveStreamBufferError } from "./LiveStreamBudget.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import * as EffectWorker from "./EffectWorker.ts";
import * as EventSink from "./EventSink.ts";
import * as EventStore from "./EventStore.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
import * as ProjectionReplay from "./testkit/ProjectionReplay.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ProjectStore from "./ProjectStore.ts";
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
const isLiveStreamBufferError = Schema.is(LiveStreamBufferError);
const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
const layerDatabase = SqlitePersistence.layerMemory;
const layerEventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(layerDatabase));
const layerProjectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(layerDatabase));
const layerStoresProvided = Layer.mergeAll(
layerDatabase,
layerEventStoreProvided,
layerProjectionStoreProvided,
);
const layerEventSinkProvided = EventSink.layer.pipe(Layer.provide(layerStoresProvided));
const layerEffectOutboxProvided = EffectOutbox.layer.pipe(Layer.provide(layerDatabase));
const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe(
Layer.provide(layerDatabase),
);
const layerIsolatedOutbox = Layer.fresh(EffectOutbox.layer.pipe(Layer.provideMerge(layerDatabase)));
const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe(
Layer.provide(layerDatabase),
);
const layerProjectionReplayProvided = ProjectionReplay.layer.pipe(
Layer.provide(layerStoresProvided),
);
const layerTest = Layer.mergeAll(
layerStoresProvided,
layerEventSinkProvided,
layerEffectOutboxProvided,
layerCommandReceiptStoreProvided,
IdAllocator.layer,
layerProjectionMaintenanceProvided,
layerProjectionReplayProvided,
);
const providerInstanceId = ProviderInstanceId.make("codex");
const providerDriver = ProviderDriverKind.make("codex");
const modelSelection = {
instanceId: providerInstanceId,
model: "gpt-5.4",
} satisfies ModelSelection;
function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread {
return {
createdBy: "user",
creationSource: "web",
id: threadId,
projectId: ProjectId.make(`project:${threadId}`),
title: `Thread ${threadId}`,
providerInstanceId,
modelSelection,
runtimeMode: "full-access",
interactionMode: "default",
branch: null,
worktreePath: null,
activeProviderThreadId: null,
lineage: {
parentThreadId: null,
relationshipToParent: null,
rootThreadId: threadId,
},
forkedFrom: null,
createdAt: now,
updatedAt: now,
archivedAt: null,
settledOverride: null,
settledAt: null,
lastVisitedAt: null,
deletedAt: null,
};
}
function threadCreatedEvent(input: {
readonly id: string;
readonly thread: OrchestrationV2AppThread;
readonly now: DateTime.Utc;
}): OrchestrationV2DomainEvent {
return {
id: EventId.make(input.id),
type: "thread.created",
threadId: input.thread.id,
providerInstanceId,
occurredAt: input.now,
payload: input.thread,
};
}
const seedCompaction = Effect.fnUntraced(function* () {
const sql = yield* SqlClient.SqlClient;
yield* sql`INSERT INTO orchestration_v2_legacy_imports (
thread_id, source_updated_at, shell_imported_at, transcript_imported_at, imported_message_count
) VALUES ('imported', '2026-10-17', '2026-10-17', '2026-10-17', 0)`;
yield* sql`WITH RECURSIVE history(n) AS (
SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 151
) INSERT INTO orchestration_events (
event_id, aggregate_kind, stream_id, stream_version, event_type,
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
) SELECT 'event-' || n, 'thread', 'thread', n,
CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END,
'2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history`;
// A pass a day ago marked the whole log; projections cover it.
yield* sql`INSERT INTO orchestration_v2_projection_metadata (
projection_name, schema_version, last_sequence, updated_at
) VALUES
('event-log-prune-mark', 2, 151, '1969-12-30T00:00:00.000Z'),
('thread-projections', 2, 151, '1969-12-30T00:00:00.000Z')
ON CONFLICT(projection_name) DO UPDATE SET last_sequence = excluded.last_sequence`;
yield* sql`WITH RECURSIVE history(n) AS (
SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100
) INSERT INTO orchestration_command_receipts (
rowid, command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
) SELECT n * 2, 'command-' || n, 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy' FROM history`;
});
it.effect("discovers on the reader worker and deletes on the writer", () =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const filename = path.join(yield* fs.makeTempDirectoryScoped(), "fixture.sqlite");
yield* Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
yield* seedCompaction();
// Establish the reader before recording main-thread statements.
yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
const reads: string[] = [];
const writes: string[] = [];
const all = NodeSqlite.StatementSync.prototype.all;
const run = NodeSqlite.StatementSync.prototype.run;
yield* Effect.acquireRelease(
Effect.sync(() => {
const readSpy = vi
.spyOn(NodeSqlite.StatementSync.prototype, "all")
.mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
reads.push(this.sourceSQL);
return all.apply(this, params);
});
const writeSpy = vi
.spyOn(NodeSqlite.StatementSync.prototype, "run")
.mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
writes.push(this.sourceSQL);
return run.apply(this, params);
});
return { readSpy, writeSpy };
}),
({ readSpy, writeSpy }) =>
Effect.sync(() => {
readSpy.mockRestore();
writeSpy.mockRestore();
}),
);
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
const result = yield* maintenance.pruneEventStore();
assert.equal(result.deletedEventCount, 150);
assert.equal(result.deletedReceiptCount, 100);
// Only the in-transaction check for rows appended during the pass reads on the writer.
assert.deepEqual(
reads.filter((query) => /\bSELECT\b/.test(query) && !query.includes("AS prompt_sequence")),
[],
);
assert(writes.some((query) => query.includes("DELETE FROM orchestration_events")));
assert(writes.some((query) => query.includes("DELETE FROM orchestration_command_receipts")));
}).pipe(
Effect.provide(
ProjectionMaintenance.layer.pipe(
Layer.provideMerge(SqlitePersistence.layerFromPath(filename)),
),
),
);
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);
it.effect("paces scheduled compaction while preserving retention state and fixed scan bounds", () =>
Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const clock = yield* Clock.Clock;
yield* seedCompaction();
const completed = yield* Deferred.make<void>();
const pauseStarted = yield* Deferred.make<void>();
let activeNanos = 0n;
let eventPages = 0;
const pauses: number[] = [];
const compactionSpans: Tracer.Span[] = [];
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options);
if (span.name === "ProjectionMaintenance.pruneEventStore") compactionSpans.push(span);
const end = span.end.bind(span);
span.end = (time, exit) => {
end(time, exit);
const query = span.attributes.get("db.query.text");
if (typeof query !== "string" || !query.includes("LIMIT")) return;
if (query.includes("FROM orchestration_events AS event")) {
eventPages++;
} else if (!query.includes("FROM orchestration_command_receipts AS receipt")) return;
// Model a page exceeding the active budget, without wall-clock sleeps.
activeNanos += 30_000_000n;
};
return span;
},
});
yield* Effect.gen(function* () {
yield* DatabaseStorage.DatabaseStorage;
yield* TestClock.adjust("1 minute");
yield* Effect.raceFirst(Deferred.await(pauseStarted), Deferred.await(completed));
yield* TestClock.adjust("269 millis");
assert.equal(eventPages, 1);
assert.deepEqual(pauses, [270]);
assert.deepEqual(
compactionSpans.map((span) => span.status._tag),
["Started"],
);
// Foreground commits must not extend the compactor's initial scan bounds.
yield* sql`INSERT INTO orchestration_events (
event_id, aggregate_kind, stream_id, stream_version, event_type,
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
) VALUES ('after-start', 'thread', 'thread', 152, 'thread.visited', '2026-10-17', 'server', '{}', '{}', 2)`;
yield* sql`INSERT INTO orchestration_command_receipts (
command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
) VALUES ('after-start', 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy')`;
yield* TestClock.adjust("10 seconds");
yield* Deferred.await(completed);
}).pipe(
Effect.provide(
DatabaseStorage.layer.pipe(
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "compaction-" })),
Layer.provide(NodeServices.layer),
),
),
Effect.withTracer(tracer),
Effect.provideService(Clock.Clock, {
...clock,
monotonicTimeNanos: Effect.sync(() => activeNanos),
sleep: (duration) => {
const millis = Duration.toMillis(duration);
if (millis < 60_000) {
pauses.push(millis);
Deferred.doneUnsafe(pauseStarted, Effect.void);
} else if (millis === 6 * 60 * 60_000) {
Deferred.doneUnsafe(completed, Effect.void);
}
return clock.sleep(duration);
},
}),
);
assert.deepEqual(pauses, Array(8).fill(270));
assert.deepEqual(
compactionSpans.map((span) => span.status._tag),
["Ended"],
);
// 151 stopped being the stream head when 152 committed mid-pass; 152 is past the bound.
assert.deepEqual(yield* sql`SELECT sequence FROM orchestration_events ORDER BY sequence`, [
{ sequence: 152 },
]);
assert.deepEqual(yield* sql`SELECT command_id FROM orchestration_command_receipts`, [
{ command_id: "after-start" },
]);
}).pipe(Effect.scoped, Effect.provide(layerTest)),
);
it.effect(
"commits an ordered provider batch atomically and publishes only after the outer commit",
() =>
Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const store = yield* EventStore.EventStoreV2;
const now = yield* DateTime.now;
const thread = makeThread(ThreadId.make("thread:single-stream-batch"), now);
yield* sink.write({ events: [threadCreatedEvent({ id: "batch-created", thread, now })] });
const message = (id: string): OrchestrationV2DomainEvent => ({
id: EventId.make(id),
type: "message.updated",
threadId: thread.id,
occurredAt: now,
payload: {
id: MessageId.make("batch-message"),
threadId: thread.id,
runId: null,
nodeId: null,
createdBy: "agent",
creationSource: "provider",
role: "assistant",
text: id,
streaming: true,
attachments: [],
createdAt: now,
updatedAt: now,
},
});
const highWater = yield* sink.latestSequence();
const callerError = { _tag: "CallerError" } as const;
assert.strictEqual(
yield* sink.withProviderBatch(Effect.fail(callerError)).pipe(Effect.flip),
callerError,
);
const failed = yield* sink
.withProviderBatch(
Effect.gen(function* () {
yield* sink.write({ events: [message("batch-duplicate")] });
yield* sink.write({ events: [message("batch-duplicate")] });
}),
)
.pipe(Effect.exit);
assert.isTrue(Exit.isFailure(failed));
assert.equal(yield* sink.latestSequence(), highWater);
assert.isEmpty((yield* projections.getThreadRecords(thread.id, ["messages"])).messages);
const firstPublished = yield* Deferred.make<void>();
const writesApplied = yield* Deferred.make<void>();
const release = yield* Deferred.make<void>();
const live = yield* sink.stream({ afterSequence: highWater }).pipe(
Stream.tap(() => Deferred.succeed(firstPublished, undefined)),
Stream.take(2),
Stream.runCollect,
Effect.forkScoped({ startImmediately: true }),
);
const batch = yield* sink
.withProviderBatch(
Effect.gen(function* () {
yield* sink.write({ events: [message("batch-first")] });
yield* sink.write({ events: [message("batch-second")] });
assert.equal(
(yield* projections.getThreadRecords(thread.id, ["messages"])).messages[0]?.text,
"batch-second",
);
yield* Deferred.succeed(writesApplied, undefined);
yield* Deferred.await(release);
}),
)
.pipe(Effect.forkScoped({ startImmediately: true }));
yield* Deferred.await(writesApplied);
yield* TestClock.adjust(0);
assert.isFalse(yield* Deferred.isDone(firstPublished));
assert.isUndefined(yield* Effect.sync(() => batch.pollUnsafe()));
yield* Deferred.succeed(release, undefined);
yield* Fiber.join(batch);
const published = yield* Fiber.join(live);
assert.deepEqual(
published.map((stored) => stored.event.id),
["batch-first", "batch-second"],
);
assert.deepEqual(
yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect),
published,
);
}).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped),
);
it.effect.each(
(["none", "duplicate", "after-write"] as const).flatMap((failure) =>
[1, 10].map((streams) => ({ failure, streams })),
),
)(
"shares the writer and isolates failure=$failure across $streams independent consumers",
({ failure, streams }) => {
const rollback = failure !== "none";
const failedIndex = streams - 1;
const jobError = { _tag: "InvalidUpdate" } as const;
const writes: number[] = [];
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (time, exit) => {
end(time, exit);
if (span.name === "orchestrationV2.EventSink.providerBatch" && Exit.isSuccess(exit)) {
writes.push(Number(span.attributes.get("orchestration_v2.batch_writes")));
}
};
return span;
},
});
return Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const threads = Array.from({ length: streams }, (_, index) =>
makeThread(ThreadId.make(`shared:${index}`), now),
);
yield* sink.write({
events: threads.map((thread, index) =>
threadCreatedEvent({ id: `created:${index}`, thread, now }),
),
});
const highWater = yield* sink.latestSequence();
const store = yield* EventStore.EventStoreV2;
const publications = yield* Queue.unbounded<EventId>();
yield* sink.stream({ afterSequence: highWater }).pipe(
Stream.runForEach((stored) => Queue.offer(publications, stored.event.id)),
Effect.forkScoped({ startImmediately: true }),
);
const pulled = yield* Deferred.make<void>();
let arrivals = 0;
const inputs = yield* Effect.forEach(threads, () =>
Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(),
);
const consumers = yield* Effect.forEach(inputs, (input, index) =>
sink
.consumeProviderEvents(
Stream.fromQueue(input).pipe(
Stream.tap(() =>
++arrivals === streams ? Deferred.succeed(pulled, undefined) : Effect.void,
),
),
(raw) =>
raw.type === "message.updated"
? sink
.write({
events: [
{
id: EventId.make(
failure === "duplicate" && index === failedIndex
? "created:0"
: `shared-event:${index}`,
),
threadId: raw.message.threadId,
occurredAt: now,
type: "message.updated",
payload: raw.message,
},
],
})
.pipe(
Effect.andThen(
failure === "after-write" && index === failedIndex
? Effect.fail(jobError)
: Effect.void,
),
)
: Effect.void,
Effect.succeed(false),
Effect.succeed(true),
)
.pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })),
);
for (let index = 0; index < threads.length; index++) {
yield* Queue.offer(inputs[index]!, {
type: "message.updated",
driver: ProviderDriverKind.make("codex"),
message: {
id: MessageId.make(`message:${index}`),
threadId: threads[index]!.id,
runId: null,
nodeId: null,
createdBy: "agent",
creationSource: "provider",
role: "assistant",
text: `stream:${index}`,
streaming: true,
attachments: [],
createdAt: now,
updatedAt: now,
},
});
}
yield* Deferred.await(pulled);
yield* TestClock.adjust(0);
yield* sink.write({ events: [] });
if (rollback) {
const exit = yield* Fiber.join(consumers[failedIndex]!);
assert.isTrue(Exit.isFailure(exit));
if (failure === "after-write" && Exit.isFailure(exit))
assert.strictEqual(Cause.squash(exit.cause), jobError);
}
yield* TestClock.adjust(0);
const replay = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect);
assert.deepEqual(
replay.length === 0 ? [] : yield* Queue.takeN(publications, replay.length),
replay.map((stored) => stored.event.id),
);
assert.equal(yield* Queue.size(publications), 0);
assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]);
assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0));
for (let index = 0; index < threads.length; index++) {
assert.lengthOf(
(yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages,
rollback && index === failedIndex ? 0 : 1,
);
}
}).pipe(
Effect.scoped,
Effect.provide(Layer.fresh(layerTest)),
Effect.withTracer(tracer),
Effect.withTracerEnabled(true),
Effect.provideService(
ConfigProvider.ConfigProvider,
ConfigProvider.fromUnknown({ T3CODE_PROVIDER_COMMIT_BATCHING: true }),
),
);
},
);
it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
let commits = 0;
const tracer = Tracer.make({
span(options) {
const span = new Tracer.NativeSpan(options);
const end = span.end.bind(span);
span.end = (time, exit) => {
end(time, exit);
if (span.name === "COMMIT") commits++;
};
return span;
},
});
return Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const events = yield* EventStore.EventStoreV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const sql = yield* SqlClient.SqlClient;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const now = yield* DateTime.now;
const threads = Array.from({ length: 65 }, (_, i) =>
ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
);
const sharedSession = {
id: ProviderSessionId.make("shutdown-shared"),
driver: providerDriver,
providerInstanceId,
status: "ready" as const,
cwd: process.cwd(),
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
};
yield* sink.write({
events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
...(i < 2
? [
{
id: EventId.make(`shutdown-session:${i}`),
type: "provider-session.attached" as const,
threadId,
occurredAt: now,
payload: sharedSession,
},
]
: []),
{
id: EventId.make(`shutdown-provider:${i}`),
type: "provider-thread.updated",
threadId,
occurredAt: now,
payload: {
id: ProviderThreadId.make(`shutdown-provider:${i}`),
driver: providerDriver,
providerInstanceId,
providerSessionId: i < 2 ? sharedSession.id : null,
appThreadId: threadId,
ownerNodeId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: i === 1 || i === 32 ? "idle" : "active",
firstRunOrdinal: null,
lastRunOrdinal: null,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
},
},
]),
});
yield* outbox.enqueue(
threads.map((threadId, i) => ({
id: `shutdown-effect:${i}`,
commandId: CommandId.make(`shutdown-command:${i}`),
threadId,
request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
})),
);
let eventlessNotifications = 0;
let armed = fail;
const recovery = yield* ProviderRuntimeRecovery.make.pipe(
Effect.provide(ServerSettings.layerTest()),
Effect.provideService(EffectOutbox.EffectOutboxV2, {
...outbox,
signalCancellations: (ids) =>
Effect.gen(function* () {
assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
eventlessNotifications += ids.length;
yield* outbox.signalCancellations(ids);
}),
}),
Effect.provideService(
EffectWorker.OrchestrationEffectWorkerV2,
EffectWorker.OrchestrationEffectWorkerV2.of({
awaitWork: Effect.void,
runRecoveryOnce: Effect.succeed(false),
runOnce: Effect.succeed(false),
nextClaimableAt: Effect.succeedNone,
drain: () => Effect.succeed(0),
}),
),
Effect.provideService(EventSink.EventSinkV2, {
...sink,
commitCommand: (input) =>
armed && input.threadId === threads[33]
? Effect.die("injected shutdown failure")
: sink.commitCommand(input),
}),
);
commits = 0;
const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
assert.equal(Exit.isFailure(result), fail);
assert.equal(commits, fail ? 1 : 3);
const receipts =
yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
assert.deepEqual(
receipts.map((r) => r.aggregate_id),
threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
);
assert.deepEqual(
yield* projections.getRecoveryThreadIds("runtime"),
fail ? threads.slice(32) : [],
);
assert.equal(eventlessNotifications, fail ? 1 : 2);
for (const [i] of threads.entries()) {
const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
}
assert.equal(
(yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
(e) =>
e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
).length,
1,
);
armed = false;
yield* recovery.reconcile("shutdown");
assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
const highWater = yield* sink.latestSequence();
yield* recovery.reconcile("shutdown");
assert.equal(yield* sink.latestSequence(), highWater);
}).pipe(
Effect.provide(Layer.fresh(layerTest)),
Effect.withTracer(tracer),
Effect.withTracerEnabled(true),
Effect.provideService(Statement.SpanPropagationEnabled, true),
);
});
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
Effect.scoped(
Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const store = yield* EventStore.EventStoreV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const thread = makeThread(ThreadId.make("thread:large-stream-body"), now);
const output = {
content: [{ type: "text", text: "first line\n" + "x".repeat(9 * 1024 * 1024) }],
};
const tool: OrchestrationV2TurnItem = {
id: TurnItemId.make("tool:large-stream-body"),
type: "dynamic_tool",
threadId: thread.id,
runId: null,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 1,
status: "running",
title: "Large tool",
toolName: "mcp__large_tool",
input: { query: "details" },
output,
startedAt: now,
completedAt: null,
updatedAt: now,
};
const [created] = yield* sink.write({
events: [
threadCreatedEvent({ id: "event:large-stream-body:created", thread, now }),
{
id: EventId.make("event:large-stream-body:replay"),
type: "turn-item.updated",
threadId: thread.id,
occurredAt: now,
payload: tool,
},
],
});
const subscribe = () =>
Stream.toPull(
sink.stream({
threadId: thread.id,
afterSequence: created!.sequence,
bounded: true,
}),
);
const pull = yield* subscribe();
const other = yield* subscribe();
const replay = yield* pull;
assert.lengthOf(replay, 1);
assert.lengthOf(yield* other, 1);
assert.equal(replay[0]!.event.type, "turn-item.updated");
assert.isBelow(Buffer.byteLength(yield* encodeJson(replay)), 4_000);
const stringify = yield* Effect.acquireRelease(
Effect.sync(() => vi.spyOn(JSON, "stringify")),
(spy) => Effect.sync(() => spy.mockRestore()),
);
const [completed] = yield* sink.write({
events: [
{
id: EventId.make("event:large-stream-body:live"),
type: "turn-item.updated",
threadId: thread.id,
occurredAt: now,
payload: { ...tool, status: "completed", completedAt: now },
},
],
});
const live = yield* pull;
const otherLive = yield* other;
assert.deepEqual(
live.map((stored) => stored.sequence),
[completed!.sequence],
);
// Both subscribers get one shared projection, so the live budget sizes it once.
assert.strictEqual(otherLive[0], live[0]);
assert.lengthOf(
stringify.mock.calls.filter(([value]) => value === live[0] || value === otherLive[0]),
1,
);
assert.isBelow(Buffer.byteLength(yield* encodeJson(live)), 4_000);
for (const stored of [...replay, ...live]) {
assert.equal(stored.event.type, "turn-item.updated");
if (stored.event.type !== "turn-item.updated") return;
assert.equal(stored.event.payload.type, "dynamic_tool");
if (stored.event.payload.type !== "dynamic_tool") return;
assert.notProperty(stored.event.payload, "output");
}
const persisted = yield* store.read({ threadId: thread.id }).pipe(Stream.runCollect);
const fullEvent = persisted.find(
(stored) => stored.sequence === completed!.sequence,
)!.event;
assert.equal(fullEvent.type, "turn-item.updated");
if (fullEvent.type !== "turn-item.updated") return;
assert.equal(fullEvent.payload.type, "dynamic_tool");
if (fullEvent.payload.type !== "dynamic_tool") return;
assert.deepEqual(fullEvent.payload.output, output);
const detail = (yield* projections.getThreadProjection(thread.id)).turnItems.find(
(item) => item.id === tool.id,
)!;
assert.equal(detail.type, "dynamic_tool");
if (detail.type === "dynamic_tool") assert.deepEqual(detail.output, output);
}),
),
);
it.effect.each(["high-water", "replay"] as const)(
"bounds live events while the V2 %s query is blocked",
(phase) =>
Effect.scoped(
Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const thread = makeThread(ThreadId.make(`thread:blocked-${phase}`), now);
const [created] = yield* sink.write({
events: [threadCreatedEvent({ id: `event:blocked-${phase}:created`, thread, now })],
});
const readStarted = yield* Deferred.make<void>();
const readClosed = yield* Deferred.make<void>();
const blockRead: Statement.Transformer = (statement) => {
const [query] = statement.compile();
if (
query.includes("FROM orchestration_events") &&
query.includes("MAX(sequence)") === (phase === "high-water")
) {
return Deferred.succeed(readStarted, undefined).pipe(
Effect.andThen(Effect.never),
Effect.ensuring(Deferred.succeed(readClosed, undefined)),
Effect.as(statement),
);
}
return Effect.succeed(statement);
};
const reader = yield* sink
.stream({ threadId: thread.id, afterSequence: created!.sequence, bounded: true })
.pipe(
Stream.provideService(Statement.CurrentTransformer, blockRead),
Stream.runDrain,
Effect.result,
Effect.forkScoped,
);
yield* Deferred.await(readStarted);
yield* sink.write({
events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
id: EventId.make(`event:blocked-${phase}:${index}`),
type: "thread.metadata-updated" as const,
threadId: thread.id,
occurredAt: now,
payload: { ...thread, title: `Updated ${index}` },
})),
});
yield* Deferred.await(readClosed);
const result = yield* Fiber.join(reader);
assert.equal(result._tag, "Failure");
if (result._tag === "Failure") {
assert.equal(result.failure._tag, "EventSinkStreamError");
assert.isTrue(isLiveStreamBufferError(result.failure.cause));
}
}),
),
);
it.effect(
"keeps internal streams subscribed while replay is blocked beyond the RPC buffer cap",
() =>
Effect.scoped(
Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const thread = makeThread(ThreadId.make("thread:internal-stream-burst"), now);
const [created] = yield* sink.write({
events: [
threadCreatedEvent({ id: "event:internal-stream-burst:created", thread, now }),
],
});
const readStarted = yield* Deferred.make<void>();
const releaseRead = yield* Deferred.make<void>();
const blockHighWater: Statement.Transformer = (statement) => {
const [query] = statement.compile();
return query.includes("MAX(sequence)") && query.includes("FROM orchestration_events")
? Deferred.succeed(readStarted, undefined).pipe(
Effect.andThen(Deferred.await(releaseRead)),
Effect.as(statement),
)
: Effect.succeed(statement);
};
const eventCount = LIVE_STREAM_MAX_ITEMS + 1;
const reader = yield* sink
.stream({ threadId: thread.id, afterSequence: created!.sequence })
.pipe(
Stream.provideService(Statement.CurrentTransformer, blockHighWater),
Stream.take(eventCount),
Stream.runCollect,
Effect.forkScoped,
);
yield* Deferred.await(readStarted);
const written = yield* sink.write({
events: Array.from({ length: eventCount }, (_, index) => ({
id: EventId.make(`event:internal-stream-burst:${index}`),
type: "thread.metadata-updated" as const,
threadId: thread.id,
occurredAt: now,
payload: { ...thread, title: `Updated ${index}` },
})),
});
yield* Deferred.succeed(releaseRead, undefined);
assert.deepEqual(
(yield* Fiber.join(reader)).map((event) => event.sequence),
written.map((event) => event.sequence),
);
}),
),
);
it.effect("filters worker replay and live queues without losing matching events", () =>
Effect.scoped(
Effect.gen(function* () {
const sink = yield* EventSink.EventSinkV2;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const thread = makeThread(ThreadId.make("thread:filtered-worker"), now);
const other = makeThread(ThreadId.make("thread:filtered-worker-other"), now);
const run: OrchestrationV2Run = {
id: RunId.make("run:filtered-worker"),
threadId: thread.id,
ordinal: 1,
providerInstanceId,
modelSelection,
providerThreadId: null,
userMessageId: MessageId.make("message:filtered-worker"),
rootNodeId: null,
activeAttemptId: null,
status: "completed",
queuePosition: null,
requestedAt: now,
startedAt: now,
completedAt: now,
checkpointId: null,
contextHandoffId: null,
};
const runEvent = (id: string, payload: OrchestrationV2Run): OrchestrationV2DomainEvent => ({
id: EventId.make(id),
type: "run.updated",
threadId: payload.threadId,
runId: payload.id,
occurredAt: now,
payload,
});
const history = yield* sink.write({
events: [
threadCreatedEvent({ id: "event:filtered-worker:created", thread, now }),
threadCreatedEvent({ id: "event:filtered-worker:other", thread: other, now }),
runEvent("event:filtered-worker:history", run),
runEvent("event:filtered-worker:other-history", {
...run,
id: RunId.make("run:filtered-worker-other"),
threadId: other.id,
}),
],
});
const original = yield* sql<{ readonly payload_json: string }>`
SELECT payload_json FROM orchestration_events
WHERE sequence = ${history[0]!.sequence}
`;
yield* Effect.acquireRelease(
sql`
UPDATE orchestration_events SET payload_json = 'unreadable unrelated payload'
WHERE sequence = ${history[0]!.sequence}
`,
() =>
sql`
UPDATE orchestration_events SET payload_json = ${original[0]!.payload_json}
WHERE sequence = ${history[0]!.sequence}
`.pipe(Effect.orDie),
);
const pull = yield* Stream.toPull(
sink.stream({ threadId: thread.id, eventType: "run.updated" }),
);
assert.deepEqual(
(yield* pull).map((stored) => stored.sequence),
[history[2]!.sequence],
);
yield* sink.write({
events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
id: EventId.make(`event:filtered-worker:output:${index}`),
type: "thread.metadata-updated" as const,
threadId: thread.id,
occurredAt: now,
payload: { ...thread, title: `Output ${index}` },
})),
});
const live = yield* sink.write({
events: [
runEvent("event:filtered-worker:live:1", { ...run, status: "interrupted" }),
runEvent("event:filtered-worker:live:2", { ...run, status: "failed" }),
],
});
assert.deepEqual(
(yield* pull).map((stored) => stored.sequence),
live.map((stored) => stored.sequence),
);
}),
),
);
it.effect("paginates catch-up beyond the event-store read limit", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:foundation-large-catch-up");
const thread = makeThread(threadId, now);
const eventCount = 1_005;
const events: Array<OrchestrationV2DomainEvent> = [
threadCreatedEvent({ id: "event:foundation-catch-up:0", thread, now }),
...Array.from({ length: eventCount - 1 }, (_, index) => ({
id: EventId.make(`event:foundation-catch-up:${index + 1}`),
type: "thread.metadata-updated" as const,
threadId,
providerInstanceId,
occurredAt: now,
payload: {
...thread,
title: `Catch-up update ${index + 1}`,
},
})),
];
yield* eventSink.write({ events });
const replayed = yield* eventSink.stream({ afterSequence: 0 }).pipe(
Stream.take(eventCount),
Stream.runCollect,
Effect.map((events) => Array.from(events)),
);
assert.lengthOf(replayed, eventCount);
assert.deepEqual(
replayed.map((stored) => stored.sequence),
Array.from({ length: eventCount }, (_, index) => index + 1),
);
}),
);
it.effect("does not lose or duplicate events while transitioning from catch-up to live", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const threadId = ThreadId.make("thread:foundation-stream-race");
const thread = makeThread(threadId, now);
const created = yield* eventSink.write({
events: [threadCreatedEvent({ id: "event:foundation-stream-race:0", thread, now })],
});
let afterSequence = created[0]!.sequence;
// oxlint-disable-next-line t3code/no-polling-loop-in-tests -- Fixed-count writes exercise the catch-up/live handoff; each iteration awaits its event rather than polling.
for (let index = 1; index <= 32; index += 1) {
const nextEvent = {
id: EventId.make(`event:foundation-stream-race:${index}`),
type: "thread.metadata-updated" as const,
threadId,
providerInstanceId,
occurredAt: now,
payload: { ...thread, title: `Race update ${index}` },
} satisfies OrchestrationV2DomainEvent;
const reader = yield* eventSink
.stream({ threadId, afterSequence })
.pipe(Stream.runHead, Effect.forkChild);
yield* Effect.yieldNow;
const written = yield* eventSink.write({ events: [nextEvent] });
const received = yield* Fiber.join(reader);
if (Option.isNone(received)) {
return yield* Effect.die("The event stream ended before delivering the live event.");
}
assert.equal(received.value.sequence, written[0]?.sequence);
assert.equal(received.value.event.id, nextEvent.id);
afterSequence = received.value.sequence;
}
}),
);
it.effect("replays shared provider-session payloads across every bound thread", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const maintenance = yield* ProjectionReplay.ProjectionReplay;
const now = yield* DateTime.now;
const firstThreadId = ThreadId.make("thread:foundation-shared-session:first");
const secondThreadId = ThreadId.make("thread:foundation-shared-session:second");
const providerSessionId = ProviderSessionId.make("provider-session:foundation:shared");
const firstSession = {
id: providerSessionId,
driver: providerDriver,
providerInstanceId,
status: "ready" as const,
cwd: "/workspace/first",
model: modelSelection.model,
capabilities: CodexProviderCapabilitiesV2,
createdAt: now,
updatedAt: now,
lastError: null,
};
const secondSession = { ...firstSession, cwd: "/workspace/second" };
yield* eventSink.write({
events: [
threadCreatedEvent({
id: "event:foundation-shared-session:first-thread",
thread: makeThread(firstThreadId, now),
now,
}),
{
id: EventId.make("event:foundation-shared-session:first-attachment"),
type: "provider-session.attached",
threadId: firstThreadId,
driver: providerDriver,
providerInstanceId,
occurredAt: now,
payload: firstSession,
},
threadCreatedEvent({
id: "event:foundation-shared-session:second-thread",
thread: makeThread(secondThreadId, now),
now,
}),
{
id: EventId.make("event:foundation-shared-session:second-attachment"),
type: "provider-session.attached",
threadId: secondThreadId,
driver: providerDriver,
providerInstanceId,
occurredAt: now,
payload: secondSession,
},
],
});
assert.equal(
(yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.cwd,
secondSession.cwd,
);
assert.isTrue((yield* maintenance.verify).valid);
assert.isTrue((yield* maintenance.rebuild).valid);
}),
);
it.effect("prunes covered history, imported v1 events, and legacy receipts", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
const sql = yield* SqlClient.SqlClient;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const now = yield* DateTime.now;
const nowIso = DateTime.formatIso(now);
const threadId = ThreadId.make("thread:foundation-compact");
const thread = makeThread(threadId, now);
const messageId = MessageId.make("message:foundation-compact");
const threadStateEvent = (
suffix: string,
type: "thread.visited" | "thread.metadata-updated" | "thread.tags-updated",
): OrchestrationV2DomainEvent => ({
id: EventId.make(`event:foundation-compact:${suffix}`),
type,
threadId,
providerInstanceId,
occurredAt: now,
payload: { ...thread, lastVisitedAt: now },
});
const messageEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
id: EventId.make(`event:foundation-compact:${suffix}`),
type: "message.updated",
threadId,
occurredAt: now,
payload: {
createdBy: "user",
creationSource: "web",
id: messageId,
threadId,
runId: null,
nodeId: null,
role: "user",
text,
attachments: [],
streaming: false,
createdAt: now,
updatedAt: now,
},
});
const nodeId = NodeId.make("node:foundation-compact");
const nodeEvent = (
suffix: string,
status: "running" | "completed",
): OrchestrationV2DomainEvent => ({
id: EventId.make(`event:foundation-compact:${suffix}`),
type: "node.updated",
threadId,
occurredAt: now,
payload: {
id: nodeId,
threadId,
runId: null,
parentNodeId: null,
rootNodeId: nodeId,
kind: "assistant_message",
status,
countsForRun: false,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
runtimeRequestId: null,
checkpointScopeId: null,
startedAt: now,
completedAt: status === "completed" ? now : null,
},
});
const itemEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
id: EventId.make(`event:foundation-compact:${suffix}`),
type: "turn-item.updated",
threadId,
occurredAt: now,
payload: {
id: TurnItemId.make("turn-item:foundation-compact"),
threadId,
runId: null,
nodeId: null,
providerThreadId: null,
providerTurnId: null,
nativeItemRef: null,
parentItemId: null,
ordinal: 1,
status: "completed",
title: null,
startedAt: now,
completedAt: now,
updatedAt: now,
type: "assistant_message",
messageId,
text,
streaming: false,
},
});
yield* eventSink.write({
events: [
threadCreatedEvent({ id: "event:foundation-compact:create", thread, now }),
threadStateEvent("meta", "thread.metadata-updated"),
threadStateEvent("visit-1", "thread.visited"),
messageEvent("message-1", "streaming"),
nodeEvent("node-1", "running"),
itemEvent("item-1", "streaming"),
itemEvent("item-middle", "still streaming"),
...Array.from({ length: 501 }, (_, index) =>
threadStateEvent(`history-${index}`, "thread.visited"),
),
threadStateEvent("tags-1", "thread.tags-updated"),
threadStateEvent("visit-2", "thread.visited"),
messageEvent("message-2", "final"),
nodeEvent("node-2", "completed"),
itemEvent("item-2", "final"),
// Older clients skip tag events, so these must not remove the rename above.
threadStateEvent("tags-2", "thread.tags-updated"),
],
});
const beforeCompaction = yield* projections.getThreadProjection(threadId);
const importedV1ThreadId = "thread:foundation-compact-v1-imported";
const pendingV1ThreadId = "thread:foundation-compact-v1-pending";
const insertV1Event = (threadIdValue: string, version: number) => sql`
INSERT INTO orchestration_events (
event_id, aggregate_kind, stream_id, stream_version, event_type,
occurred_at, actor_kind, payload_json, metadata_json, application_event_version
)
VALUES (
${`event:v1:${threadIdValue}:${version}`}, 'thread', ${threadIdValue}, ${version},
'thread.message-appended', ${nowIso}, 'user', '{}', '{}', 1
)
`;
yield* insertV1Event(importedV1ThreadId, 1);
yield* insertV1Event(importedV1ThreadId, 2);
yield* insertV1Event(pendingV1ThreadId, 1);
yield* sql`
INSERT INTO orchestration_v2_legacy_imports (
thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
imported_message_count, last_error
)
VALUES
(${importedV1ThreadId}, ${nowIso}, ${nowIso}, ${nowIso}, 2, NULL),
(${pendingV1ThreadId}, ${nowIso}, ${nowIso}, NULL, 0, NULL)
ON CONFLICT(thread_id) DO NOTHING
`;
yield* sql`
INSERT INTO orchestration_command_receipts (
command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
)
VALUES
('command:foundation-compact:legacy', 'thread', ${importedV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy'),
('command:foundation-compact:pending', 'thread', ${pendingV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy')
ON CONFLICT(command_id) DO NOTHING
`;
// The first pass only marks the head and drops imported legacy rows.
const marked = yield* maintenance.pruneEventStore();
yield* TestClock.adjust("1 day");
const summary = yield* maintenance.pruneEventStore();
assert.isAtLeast(summary.deletedEventCount, 504);
assert.equal(marked.deletedReceiptCount + summary.deletedReceiptCount, 1);
const remaining = yield* sql<{ readonly event_id: string }>`
SELECT event_id
FROM orchestration_events
WHERE aggregate_kind = 'thread'
AND stream_id = ${threadId}
ORDER BY sequence ASC
`;
assert.deepEqual(
remaining.map((row) => row.event_id),
["event:foundation-compact:create", "event:foundation-compact:tags-2"],
);
const remainingV1 = yield* sql<{ readonly stream_id: string }>`
SELECT stream_id
FROM orchestration_events
WHERE application_event_version = 1
AND stream_id IN (${importedV1ThreadId}, ${pendingV1ThreadId})
`;
assert.deepEqual(
remainingV1.map((row) => row.stream_id),
[pendingV1ThreadId],
);
const remainingReceipts = yield* sql<{ readonly command_id: string }>`
SELECT command_id
FROM orchestration_command_receipts
WHERE command_id IN ('command:foundation-compact:legacy', 'command:foundation-compact:pending')
`;
assert.deepEqual(
remainingReceipts.map((row) => row.command_id),
["command:foundation-compact:pending"],
);
assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeCompaction);
}),
);
it.effect("verifies and rebuilds projections with cross-thread subagent relations", () =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
const maintenance = yield* ProjectionReplay.ProjectionReplay;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const parentThreadId = ThreadId.make("thread:foundation-cross-thread:parent");
const childThreadId = ThreadId.make("thread:foundation-cross-thread:child");
const childProviderThreadId = ProviderThreadId.make(
"provider-thread:foundation-cross-thread:child",
);
const subagentId = NodeId.make("subagent:foundation-cross-thread");
const spawnTransferId = ContextTransferId.make("transfer:foundation-cross-thread:spawn");
const resultTransferId = ContextTransferId.make("transfer:foundation-cross-thread:result");
const parentThread = makeThread(parentThreadId, now);
const childThread = {
...makeThread(childThreadId, now),
createdBy: "agent" as const,
creationSource: "provider" as const,
lineage: {
parentThreadId,
relationshipToParent: "subagent" as const,
rootThreadId: parentThreadId,
},
};
yield* eventSink.write({
events: [
threadCreatedEvent({
id: "event:foundation-cross-thread:parent",
thread: parentThread,
now,
}),
threadCreatedEvent({
id: "event:foundation-cross-thread:child",
thread: childThread,
now,
}),
{
id: EventId.make("event:foundation-cross-thread:subagent"),
type: "subagent.updated",
threadId: parentThreadId,
nodeId: subagentId,
providerInstanceId,
occurredAt: now,
payload: {
id: subagentId,
threadId: parentThreadId,
runId: null,
parentNodeId: NodeId.make("node:foundation-cross-thread:parent"),
origin: "app_owned",
createdBy: "agent",
driver: providerDriver,
providerInstanceId,
providerThreadId: childProviderThreadId,
childThreadId,
nativeTaskRef: null,
prompt: "Inspect the child flow",
title: "Cross-thread child",
model: modelSelection.model,
status: "completed",
result: "done",
startedAt: now,
completedAt: now,
updatedAt: now,
},
},
{
id: EventId.make("event:foundation-cross-thread:spawn-transfer"),
type: "context-transfer.created",
threadId: childThreadId,
providerInstanceId,
occurredAt: now,
payload: {
id: spawnTransferId,
type: "subagent_spawn",
sourceThreadId: parentThreadId,
targetThreadId: childThreadId,
sourcePoint: { threadId: parentThreadId },
basePoint: null,
sourceProviderInstanceId: providerInstanceId,
targetProviderInstanceId: providerInstanceId,
targetRunId: null,
status: "consumed",
resolution: null,
createdBy: "agent",
error: null,
createdAt: now,
updatedAt: now,
consumedAt: now,
},
},
{
id: EventId.make("event:foundation-cross-thread:provider-thread"),
type: "provider-thread.updated",
threadId: parentThreadId,
driver: providerDriver,
providerInstanceId,
occurredAt: now,
payload: {
id: childProviderThreadId,
driver: providerDriver,
providerInstanceId,
providerSessionId: null,
appThreadId: childThreadId,
ownerNodeId: null,
nativeThreadRef: null,
nativeConversationHeadRef: null,
status: "idle",
firstRunOrdinal: 1,
lastRunOrdinal: 1,
handoffIds: [],
forkedFrom: null,
createdAt: now,
updatedAt: now,
},
},
{
id: EventId.make("event:foundation-cross-thread:result-transfer"),
type: "context-transfer.created",
threadId: parentThreadId,
providerInstanceId,
occurredAt: now,
payload: {
id: resultTransferId,
type: "subagent_result",
sourceThreadId: childThreadId,
targetThreadId: parentThreadId,
sourcePoint: { threadId: childThreadId },
basePoint: null,
sourceProviderInstanceId: providerInstanceId,
targetProviderInstanceId: providerInstanceId,
targetRunId: null,
status: "consumed",
resolution: null,
createdBy: "system",
error: null,
createdAt: now,
updatedAt: now,
consumedAt: now,
},
},
],
});
const assertCrossThreadProjection = Effect.gen(function* () {
const parent = yield* projectionStore.getThreadProjection(parentThreadId);
const child = yield* projectionStore.getThreadProjection(childThreadId);
const expectedTransferIds = [spawnTransferId, resultTransferId].toSorted();
assert.deepEqual(
parent.contextTransfers.map((transfer) => transfer.id).toSorted(),
expectedTransferIds,
);
assert.deepEqual(
child.contextTransfers.map((transfer) => transfer.id).toSorted(),
expectedTransferIds,
);
assert.deepEqual(
parent.providerThreads.map((providerThread) => providerThread.id),
[childProviderThreadId],
);
assert.equal(child.thread.activeProviderThreadId, childProviderThreadId);
});
yield* assertCrossThreadProjection;
assert.isTrue((yield* maintenance.verify).valid);
yield* sql`
UPDATE orchestration_v2_projection_provider_threads SET payload_json = '{}'
WHERE provider_thread_id = ${childProviderThreadId}
`;
assert.deepEqual(
new Set((yield* maintenance.verify).unreadableThreadIds),
new Set([parentThreadId, childThreadId]),
);
assert.isTrue((yield* maintenance.rebuild).valid);
yield* assertCrossThreadProjection;
}),
);
it.effect(
"rolls back events, projections, receipts, and effects after a projection failure",
() =>
Effect.gen(function* () {
const eventSink = yield* EventSink.EventSinkV2;
const eventStore = yield* EventStore.EventStoreV2;
const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
const outbox = yield* EffectOutbox.EffectOutboxV2;
const sql = yield* SqlClient.SqlClient;
const now = yield* DateTime.now;
const commandId = CommandId.make("command:foundation-atomic-failure");
const threadId = ThreadId.make("thread:foundation-atomic-failure");
const scopeId = CheckpointScopeId.make("scope:foundation-atomic-failure");
const checkpoint = (index: number) => ({
id: CheckpointId.make(`checkpoint:foundation-atomic-failure:${index}`),
threadId,
scopeId,
runId: null,
nodeId: NodeId.make("node:foundation-atomic-failure"),
parentCheckpointId: null,
ordinalWithinScope: 1,
appRunOrdinal: null,
ref: CheckpointRef.make(`checkpoint-ref:foundation-atomic-failure:${index}`),
status: "ready" as const,
files: [],
capturedAt: now,
});
const events = [1, 2].map(
(index) =>
({
id: EventId.make(`event:foundation-atomic-failure:${index}`),
type: "checkpoint.captured",
threadId,
occurredAt: now,
payload: checkpoint(index),
}) satisfies OrchestrationV2DomainEvent,
);
const exit = yield* Effect.exit(
eventSink.commitCommand({
commandId,
threadId,
commandType: "checkpoint.atomicity-test",
acceptedAt: now,
events,
effects: [
{
id: "effect:foundation-atomic-failure",
commandId,
threadId,
request: {
type: "provider-turn.start",
runId: RunId.make("run:foundation-atomic-failure"),
},
},
],
}),
);
assert.equal(exit._tag, "Failure");
assert.isTrue(Option.isNone(yield* receipts.getByCommandId(commandId)));
assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
assert.deepEqual(
yield* eventStore.readByCommandId({ commandId }).pipe(
Stream.runCollect,
Effect.map((events) => Array.from(events)),
),
[],
);
const checkpointRows = yield* sql<{ readonly count: number }>`
SELECT COUNT(*) AS count
FROM orchestration_v2_projection_checkpoints
WHERE thread_id = ${threadId}
`;
assert.equal(checkpointRows[0]?.count, 0);
}),
);
it.effect("replays every command event across bounded persistence pages", () =>
Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const eventStore = yield* EventStore.EventStoreV2;
const eventSink = yield* EventSink.EventSinkV2;
const now = yield* DateTime.now;
const occurredAt = DateTime.formatIso(now);
const commandId = CommandId.make("command:paged-command-replay");
const exactPageCommandId = CommandId.make("command:paged-command-replay:exact");
const emptyCommandId = CommandId.make("command:paged-command-replay:empty");
const threadId = "thread:paged-command-replay";
const eventRow = (
ordinal: number,
input: {
readonly commandId?: string | null;
readonly aggregateKind?: "project" | "thread";
readonly streamId?: string;
readonly version?: number;
} = {},
) => ({
event_id: `event:paged-command-replay:${ordinal}`,
aggregate_kind: input.aggregateKind ?? "thread",
stream_id: input.streamId ?? threadId,
stream_version: ordinal,
event_type: "provider-session.detached",
occurred_at: occurredAt,
command_id: input.commandId ?? null,
causation_event_id: null,
correlation_id: null,
actor_kind: "server",
payload_json: JSON.stringify({
providerSessionId: `session:paged-command-replay:${ordinal}`,
detachedAt: occurredAt,
}),
metadata_json: "{}",
application_event_version: input.version ?? 2,
});
const matchingCount = 1_001;