sailfork / t3code-github / apps/ server/ src/ orchestration-v2/ FoundationPersistence.test.ts

import * as NodeSqlite from "node:sqlite";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { assert, it } from "@effect/vitest";
import {
  CheckpointId,
  CheckpointRef,
  CheckpointScopeId,
  CommandId,
  ContextTransferId,
  EventId,
  MessageId,
  type ModelSelection,
  NodeId,
  type OrchestrationV2AppThread,
  type OrchestrationV2DomainEvent,
  type OrchestrationV2Run,
  type OrchestrationV2ThreadProjection,
  type OrchestrationV2TurnItem,
  ProjectId,
  ProviderDriverKind,
  ProviderInstanceId,
  ProviderSessionId,
  ProviderThreadId,
  RunAttemptId,
  RunId,
  ThreadId,
  TurnItemId,
} from "@t3tools/contracts";
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import * as ConfigProvider from "effect/ConfigProvider";
import * as Queue from "effect/Queue";
import * as Arr from "effect/Array";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as DateTime from "effect/DateTime";
import * as Exit from "effect/Exit";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Ref from "effect/Ref";
import * as Scheduler from "effect/Scheduler";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import * as TestClock from "effect/testing/TestClock";
import * as Tracer from "effect/Tracer";
import * as SqlClient from "effect/sql/SqlClient";
import * as Statement from "effect/sql/Statement";

import { vi } from "vite-plus/test";

import * as ServerConfig from "../config.ts";
import * as DatabaseStorage from "../persistence/DatabaseStorage.ts";
import * as ServerSettings from "../serverSettings.ts";
import { LIVE_STREAM_MAX_ITEMS, LiveStreamBufferError } from "./LiveStreamBudget.ts";
import * as SqlitePersistence from "../persistence/Sqlite.ts";
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import * as EffectWorker from "./EffectWorker.ts";
import * as EventSink from "./EventSink.ts";
import * as EventStore from "./EventStore.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
import * as ProjectionReplay from "./testkit/ProjectionReplay.ts";
import * as ProjectionStore from "./ProjectionStore.ts";
import * as ProjectStore from "./ProjectStore.ts";
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";

const isLiveStreamBufferError = Schema.is(LiveStreamBufferError);
const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));

const layerDatabase = SqlitePersistence.layerMemory;
const layerEventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(layerDatabase));
const layerProjectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(layerDatabase));
const layerStoresProvided = Layer.mergeAll(
  layerDatabase,
  layerEventStoreProvided,
  layerProjectionStoreProvided,
);
const layerEventSinkProvided = EventSink.layer.pipe(Layer.provide(layerStoresProvided));
const layerEffectOutboxProvided = EffectOutbox.layer.pipe(Layer.provide(layerDatabase));
const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe(
  Layer.provide(layerDatabase),
);
const layerIsolatedOutbox = Layer.fresh(EffectOutbox.layer.pipe(Layer.provideMerge(layerDatabase)));
const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe(
  Layer.provide(layerDatabase),
);
const layerProjectionReplayProvided = ProjectionReplay.layer.pipe(
  Layer.provide(layerStoresProvided),
);
const layerTest = Layer.mergeAll(
  layerStoresProvided,
  layerEventSinkProvided,
  layerEffectOutboxProvided,
  layerCommandReceiptStoreProvided,
  IdAllocator.layer,
  layerProjectionMaintenanceProvided,
  layerProjectionReplayProvided,
);

const providerInstanceId = ProviderInstanceId.make("codex");
const providerDriver = ProviderDriverKind.make("codex");
const modelSelection = {
  instanceId: providerInstanceId,
  model: "gpt-5.4",
} satisfies ModelSelection;

function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread {
  return {
    createdBy: "user",
    creationSource: "web",
    id: threadId,
    projectId: ProjectId.make(`project:${threadId}`),
    title: `Thread ${threadId}`,
    providerInstanceId,
    modelSelection,
    runtimeMode: "full-access",
    interactionMode: "default",
    branch: null,
    worktreePath: null,
    activeProviderThreadId: null,
    lineage: {
      parentThreadId: null,
      relationshipToParent: null,
      rootThreadId: threadId,
    },
    forkedFrom: null,
    createdAt: now,
    updatedAt: now,
    archivedAt: null,
    settledOverride: null,
    settledAt: null,
    lastVisitedAt: null,
    deletedAt: null,
  };
}

function threadCreatedEvent(input: {
  readonly id: string;
  readonly thread: OrchestrationV2AppThread;
  readonly now: DateTime.Utc;
}): OrchestrationV2DomainEvent {
  return {
    id: EventId.make(input.id),
    type: "thread.created",
    threadId: input.thread.id,
    providerInstanceId,
    occurredAt: input.now,
    payload: input.thread,
  };
}

const seedCompaction = Effect.fnUntraced(function* () {
  const sql = yield* SqlClient.SqlClient;
  yield* sql`INSERT INTO orchestration_v2_legacy_imports (
    thread_id, source_updated_at, shell_imported_at, transcript_imported_at, imported_message_count
  ) VALUES ('imported', '2026-10-17', '2026-10-17', '2026-10-17', 0)`;
  yield* sql`WITH RECURSIVE history(n) AS (
    SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 151
  ) INSERT INTO orchestration_events (
    event_id, aggregate_kind, stream_id, stream_version, event_type,
    occurred_at, actor_kind, payload_json, metadata_json, application_event_version
  ) SELECT 'event-' || n, 'thread', 'thread', n,
    CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END,
    '2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history`;
  // A pass a day ago marked the whole log; projections cover it.
  yield* sql`INSERT INTO orchestration_v2_projection_metadata (
    projection_name, schema_version, last_sequence, updated_at
  ) VALUES
    ('event-log-prune-mark', 2, 151, '1969-12-30T00:00:00.000Z'),
    ('thread-projections', 2, 151, '1969-12-30T00:00:00.000Z')
  ON CONFLICT(projection_name) DO UPDATE SET last_sequence = excluded.last_sequence`;
  yield* sql`WITH RECURSIVE history(n) AS (
    SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100
  ) INSERT INTO orchestration_command_receipts (
    rowid, command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
  ) SELECT n * 2, 'command-' || n, 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy' FROM history`;
});

it.effect("discovers on the reader worker and deletes on the writer", () =>
  Effect.gen(function* () {
    const fs = yield* FileSystem.FileSystem;
    const path = yield* Path.Path;
    const filename = path.join(yield* fs.makeTempDirectoryScoped(), "fixture.sqlite");
    yield* Effect.gen(function* () {
      const sql = yield* SqlClient.SqlClient;
      yield* seedCompaction();
      // Establish the reader before recording main-thread statements.
      yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
      const reads: string[] = [];
      const writes: string[] = [];
      const all = NodeSqlite.StatementSync.prototype.all;
      const run = NodeSqlite.StatementSync.prototype.run;
      yield* Effect.acquireRelease(
        Effect.sync(() => {
          const readSpy = vi
            .spyOn(NodeSqlite.StatementSync.prototype, "all")
            .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
              reads.push(this.sourceSQL);
              return all.apply(this, params);
            });
          const writeSpy = vi
            .spyOn(NodeSqlite.StatementSync.prototype, "run")
            .mockImplementation(function (this: NodeSqlite.StatementSync, ...params) {
              writes.push(this.sourceSQL);
              return run.apply(this, params);
            });
          return { readSpy, writeSpy };
        }),
        ({ readSpy, writeSpy }) =>
          Effect.sync(() => {
            readSpy.mockRestore();
            writeSpy.mockRestore();
          }),
      );
      const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
      const result = yield* maintenance.pruneEventStore();
      assert.equal(result.deletedEventCount, 150);
      assert.equal(result.deletedReceiptCount, 100);
      // Only the in-transaction check for rows appended during the pass reads on the writer.
      assert.deepEqual(
        reads.filter((query) => /\bSELECT\b/.test(query) && !query.includes("AS prompt_sequence")),
        [],
      );
      assert(writes.some((query) => query.includes("DELETE FROM orchestration_events")));
      assert(writes.some((query) => query.includes("DELETE FROM orchestration_command_receipts")));
    }).pipe(
      Effect.provide(
        ProjectionMaintenance.layer.pipe(
          Layer.provideMerge(SqlitePersistence.layerFromPath(filename)),
        ),
      ),
    );
  }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
);

it.effect("paces scheduled compaction while preserving retention state and fixed scan bounds", () =>
  Effect.gen(function* () {
    const sql = yield* SqlClient.SqlClient;
    const clock = yield* Clock.Clock;
    yield* seedCompaction();
    const completed = yield* Deferred.make<void>();
    const pauseStarted = yield* Deferred.make<void>();
    let activeNanos = 0n;
    let eventPages = 0;
    const pauses: number[] = [];
    const compactionSpans: Tracer.Span[] = [];
    const tracer = Tracer.make({
      span(options) {
        const span = new Tracer.NativeSpan(options);
        if (span.name === "ProjectionMaintenance.pruneEventStore") compactionSpans.push(span);
        const end = span.end.bind(span);
        span.end = (time, exit) => {
          end(time, exit);
          const query = span.attributes.get("db.query.text");
          if (typeof query !== "string" || !query.includes("LIMIT")) return;
          if (query.includes("FROM orchestration_events AS event")) {
            eventPages++;
          } else if (!query.includes("FROM orchestration_command_receipts AS receipt")) return;
          // Model a page exceeding the active budget, without wall-clock sleeps.
          activeNanos += 30_000_000n;
        };
        return span;
      },
    });
    yield* Effect.gen(function* () {
      yield* DatabaseStorage.DatabaseStorage;
      yield* TestClock.adjust("1 minute");
      yield* Effect.raceFirst(Deferred.await(pauseStarted), Deferred.await(completed));
      yield* TestClock.adjust("269 millis");
      assert.equal(eventPages, 1);
      assert.deepEqual(pauses, [270]);
      assert.deepEqual(
        compactionSpans.map((span) => span.status._tag),
        ["Started"],
      );
      // Foreground commits must not extend the compactor's initial scan bounds.
      yield* sql`INSERT INTO orchestration_events (
          event_id, aggregate_kind, stream_id, stream_version, event_type,
          occurred_at, actor_kind, payload_json, metadata_json, application_event_version
        ) VALUES ('after-start', 'thread', 'thread', 152, 'thread.visited', '2026-10-17', 'server', '{}', '{}', 2)`;
      yield* sql`INSERT INTO orchestration_command_receipts (
          command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
        ) VALUES ('after-start', 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy')`;
      yield* TestClock.adjust("10 seconds");
      yield* Deferred.await(completed);
    }).pipe(
      Effect.provide(
        DatabaseStorage.layer.pipe(
          Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "compaction-" })),
          Layer.provide(NodeServices.layer),
        ),
      ),
      Effect.withTracer(tracer),
      Effect.provideService(Clock.Clock, {
        ...clock,
        monotonicTimeNanos: Effect.sync(() => activeNanos),
        sleep: (duration) => {
          const millis = Duration.toMillis(duration);
          if (millis < 60_000) {
            pauses.push(millis);
            Deferred.doneUnsafe(pauseStarted, Effect.void);
          } else if (millis === 6 * 60 * 60_000) {
            Deferred.doneUnsafe(completed, Effect.void);
          }
          return clock.sleep(duration);
        },
      }),
    );
    assert.deepEqual(pauses, Array(8).fill(270));
    assert.deepEqual(
      compactionSpans.map((span) => span.status._tag),
      ["Ended"],
    );
    // 151 stopped being the stream head when 152 committed mid-pass; 152 is past the bound.
    assert.deepEqual(yield* sql`SELECT sequence FROM orchestration_events ORDER BY sequence`, [
      { sequence: 152 },
    ]);
    assert.deepEqual(yield* sql`SELECT command_id FROM orchestration_command_receipts`, [
      { command_id: "after-start" },
    ]);
  }).pipe(Effect.scoped, Effect.provide(layerTest)),
);

it.effect(
  "commits an ordered provider batch atomically and publishes only after the outer commit",
  () =>
    Effect.gen(function* () {
      const sink = yield* EventSink.EventSinkV2;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const store = yield* EventStore.EventStoreV2;
      const now = yield* DateTime.now;
      const thread = makeThread(ThreadId.make("thread:single-stream-batch"), now);
      yield* sink.write({ events: [threadCreatedEvent({ id: "batch-created", thread, now })] });
      const message = (id: string): OrchestrationV2DomainEvent => ({
        id: EventId.make(id),
        type: "message.updated",
        threadId: thread.id,
        occurredAt: now,
        payload: {
          id: MessageId.make("batch-message"),
          threadId: thread.id,
          runId: null,
          nodeId: null,
          createdBy: "agent",
          creationSource: "provider",
          role: "assistant",
          text: id,
          streaming: true,
          attachments: [],
          createdAt: now,
          updatedAt: now,
        },
      });
      const highWater = yield* sink.latestSequence();
      const callerError = { _tag: "CallerError" } as const;
      assert.strictEqual(
        yield* sink.withProviderBatch(Effect.fail(callerError)).pipe(Effect.flip),
        callerError,
      );
      const failed = yield* sink
        .withProviderBatch(
          Effect.gen(function* () {
            yield* sink.write({ events: [message("batch-duplicate")] });
            yield* sink.write({ events: [message("batch-duplicate")] });
          }),
        )
        .pipe(Effect.exit);
      assert.isTrue(Exit.isFailure(failed));
      assert.equal(yield* sink.latestSequence(), highWater);
      assert.isEmpty((yield* projections.getThreadRecords(thread.id, ["messages"])).messages);

      const firstPublished = yield* Deferred.make<void>();
      const writesApplied = yield* Deferred.make<void>();
      const release = yield* Deferred.make<void>();
      const live = yield* sink.stream({ afterSequence: highWater }).pipe(
        Stream.tap(() => Deferred.succeed(firstPublished, undefined)),
        Stream.take(2),
        Stream.runCollect,
        Effect.forkScoped({ startImmediately: true }),
      );
      const batch = yield* sink
        .withProviderBatch(
          Effect.gen(function* () {
            yield* sink.write({ events: [message("batch-first")] });
            yield* sink.write({ events: [message("batch-second")] });
            assert.equal(
              (yield* projections.getThreadRecords(thread.id, ["messages"])).messages[0]?.text,
              "batch-second",
            );
            yield* Deferred.succeed(writesApplied, undefined);
            yield* Deferred.await(release);
          }),
        )
        .pipe(Effect.forkScoped({ startImmediately: true }));
      yield* Deferred.await(writesApplied);
      yield* TestClock.adjust(0);
      assert.isFalse(yield* Deferred.isDone(firstPublished));
      assert.isUndefined(yield* Effect.sync(() => batch.pollUnsafe()));
      yield* Deferred.succeed(release, undefined);
      yield* Fiber.join(batch);
      const published = yield* Fiber.join(live);
      assert.deepEqual(
        published.map((stored) => stored.event.id),
        ["batch-first", "batch-second"],
      );
      assert.deepEqual(
        yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect),
        published,
      );
    }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped),
);

it.effect.each(
  (["none", "duplicate", "after-write"] as const).flatMap((failure) =>
    [1, 10].map((streams) => ({ failure, streams })),
  ),
)(
  "shares the writer and isolates failure=$failure across $streams independent consumers",
  ({ failure, streams }) => {
    const rollback = failure !== "none";
    const failedIndex = streams - 1;
    const jobError = { _tag: "InvalidUpdate" } as const;
    const writes: number[] = [];
    const tracer = Tracer.make({
      span(options) {
        const span = new Tracer.NativeSpan(options);
        const end = span.end.bind(span);
        span.end = (time, exit) => {
          end(time, exit);
          if (span.name === "orchestrationV2.EventSink.providerBatch" && Exit.isSuccess(exit)) {
            writes.push(Number(span.attributes.get("orchestration_v2.batch_writes")));
          }
        };
        return span;
      },
    });
    return Effect.gen(function* () {
      const sink = yield* EventSink.EventSinkV2;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const threads = Array.from({ length: streams }, (_, index) =>
        makeThread(ThreadId.make(`shared:${index}`), now),
      );
      yield* sink.write({
        events: threads.map((thread, index) =>
          threadCreatedEvent({ id: `created:${index}`, thread, now }),
        ),
      });
      const highWater = yield* sink.latestSequence();
      const store = yield* EventStore.EventStoreV2;
      const publications = yield* Queue.unbounded<EventId>();
      yield* sink.stream({ afterSequence: highWater }).pipe(
        Stream.runForEach((stored) => Queue.offer(publications, stored.event.id)),
        Effect.forkScoped({ startImmediately: true }),
      );
      const pulled = yield* Deferred.make<void>();
      let arrivals = 0;
      const inputs = yield* Effect.forEach(threads, () =>
        Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(),
      );
      const consumers = yield* Effect.forEach(inputs, (input, index) =>
        sink
          .consumeProviderEvents(
            Stream.fromQueue(input).pipe(
              Stream.tap(() =>
                ++arrivals === streams ? Deferred.succeed(pulled, undefined) : Effect.void,
              ),
            ),
            (raw) =>
              raw.type === "message.updated"
                ? sink
                    .write({
                      events: [
                        {
                          id: EventId.make(
                            failure === "duplicate" && index === failedIndex
                              ? "created:0"
                              : `shared-event:${index}`,
                          ),
                          threadId: raw.message.threadId,
                          occurredAt: now,
                          type: "message.updated",
                          payload: raw.message,
                        },
                      ],
                    })
                    .pipe(
                      Effect.andThen(
                        failure === "after-write" && index === failedIndex
                          ? Effect.fail(jobError)
                          : Effect.void,
                      ),
                    )
                : Effect.void,
            Effect.succeed(false),
            Effect.succeed(true),
          )
          .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })),
      );
      for (let index = 0; index < threads.length; index++) {
        yield* Queue.offer(inputs[index]!, {
          type: "message.updated",
          driver: ProviderDriverKind.make("codex"),
          message: {
            id: MessageId.make(`message:${index}`),
            threadId: threads[index]!.id,
            runId: null,
            nodeId: null,
            createdBy: "agent",
            creationSource: "provider",
            role: "assistant",
            text: `stream:${index}`,
            streaming: true,
            attachments: [],
            createdAt: now,
            updatedAt: now,
          },
        });
      }
      yield* Deferred.await(pulled);
      yield* TestClock.adjust(0);
      yield* sink.write({ events: [] });
      if (rollback) {
        const exit = yield* Fiber.join(consumers[failedIndex]!);
        assert.isTrue(Exit.isFailure(exit));
        if (failure === "after-write" && Exit.isFailure(exit))
          assert.strictEqual(Cause.squash(exit.cause), jobError);
      }
      yield* TestClock.adjust(0);
      const replay = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect);
      assert.deepEqual(
        replay.length === 0 ? [] : yield* Queue.takeN(publications, replay.length),
        replay.map((stored) => stored.event.id),
      );
      assert.equal(yield* Queue.size(publications), 0);
      assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]);
      assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0));
      for (let index = 0; index < threads.length; index++) {
        assert.lengthOf(
          (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages,
          rollback && index === failedIndex ? 0 : 1,
        );
      }
    }).pipe(
      Effect.scoped,
      Effect.provide(Layer.fresh(layerTest)),
      Effect.withTracer(tracer),
      Effect.withTracerEnabled(true),
      Effect.provideService(
        ConfigProvider.ConfigProvider,
        ConfigProvider.fromUnknown({ T3CODE_PROVIDER_COMMIT_BATCHING: true }),
      ),
    );
  },
);

it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
  let commits = 0;
  const tracer = Tracer.make({
    span(options) {
      const span = new Tracer.NativeSpan(options);
      const end = span.end.bind(span);
      span.end = (time, exit) => {
        end(time, exit);
        if (span.name === "COMMIT") commits++;
      };
      return span;
    },
  });
  return Effect.gen(function* () {
    const sink = yield* EventSink.EventSinkV2;
    const events = yield* EventStore.EventStoreV2;
    const projections = yield* ProjectionStore.ProjectionStoreV2;
    const sql = yield* SqlClient.SqlClient;
    const outbox = yield* EffectOutbox.EffectOutboxV2;
    const now = yield* DateTime.now;
    const threads = Array.from({ length: 65 }, (_, i) =>
      ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
    );
    const sharedSession = {
      id: ProviderSessionId.make("shutdown-shared"),
      driver: providerDriver,
      providerInstanceId,
      status: "ready" as const,
      cwd: process.cwd(),
      model: modelSelection.model,
      capabilities: CodexProviderCapabilitiesV2,
      createdAt: now,
      updatedAt: now,
      lastError: null,
    };
    yield* sink.write({
      events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
        threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
        ...(i < 2
          ? [
              {
                id: EventId.make(`shutdown-session:${i}`),
                type: "provider-session.attached" as const,
                threadId,
                occurredAt: now,
                payload: sharedSession,
              },
            ]
          : []),
        {
          id: EventId.make(`shutdown-provider:${i}`),
          type: "provider-thread.updated",
          threadId,
          occurredAt: now,
          payload: {
            id: ProviderThreadId.make(`shutdown-provider:${i}`),
            driver: providerDriver,
            providerInstanceId,
            providerSessionId: i < 2 ? sharedSession.id : null,
            appThreadId: threadId,
            ownerNodeId: null,
            nativeThreadRef: null,
            nativeConversationHeadRef: null,
            status: i === 1 || i === 32 ? "idle" : "active",
            firstRunOrdinal: null,
            lastRunOrdinal: null,
            handoffIds: [],
            forkedFrom: null,
            createdAt: now,
            updatedAt: now,
          },
        },
      ]),
    });
    yield* outbox.enqueue(
      threads.map((threadId, i) => ({
        id: `shutdown-effect:${i}`,
        commandId: CommandId.make(`shutdown-command:${i}`),
        threadId,
        request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
      })),
    );
    let eventlessNotifications = 0;
    let armed = fail;
    const recovery = yield* ProviderRuntimeRecovery.make.pipe(
      Effect.provide(ServerSettings.layerTest()),
      Effect.provideService(EffectOutbox.EffectOutboxV2, {
        ...outbox,
        signalCancellations: (ids) =>
          Effect.gen(function* () {
            assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
            eventlessNotifications += ids.length;
            yield* outbox.signalCancellations(ids);
          }),
      }),
      Effect.provideService(
        EffectWorker.OrchestrationEffectWorkerV2,
        EffectWorker.OrchestrationEffectWorkerV2.of({
          awaitWork: Effect.void,
          runRecoveryOnce: Effect.succeed(false),
          runOnce: Effect.succeed(false),
          nextClaimableAt: Effect.succeedNone,
          drain: () => Effect.succeed(0),
        }),
      ),
      Effect.provideService(EventSink.EventSinkV2, {
        ...sink,
        commitCommand: (input) =>
          armed && input.threadId === threads[33]
            ? Effect.die("injected shutdown failure")
            : sink.commitCommand(input),
      }),
    );
    commits = 0;
    const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
    assert.equal(Exit.isFailure(result), fail);
    assert.equal(commits, fail ? 1 : 3);
    const receipts =
      yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
    assert.deepEqual(
      receipts.map((r) => r.aggregate_id),
      threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
    );
    assert.deepEqual(
      yield* projections.getRecoveryThreadIds("runtime"),
      fail ? threads.slice(32) : [],
    );
    assert.equal(eventlessNotifications, fail ? 1 : 2);
    for (const [i] of threads.entries()) {
      const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
      assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
    }
    assert.equal(
      (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
        (e) =>
          e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
      ).length,
      1,
    );
    armed = false;
    yield* recovery.reconcile("shutdown");
    assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
    const highWater = yield* sink.latestSequence();
    yield* recovery.reconcile("shutdown");
    assert.equal(yield* sink.latestSequence(), highWater);
  }).pipe(
    Effect.provide(Layer.fresh(layerTest)),
    Effect.withTracer(tracer),
    Effect.withTracerEnabled(true),
    Effect.provideService(Statement.SpanPropagationEnabled, true),
  );
});

it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
  it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
    Effect.scoped(
      Effect.gen(function* () {
        const sink = yield* EventSink.EventSinkV2;
        const store = yield* EventStore.EventStoreV2;
        const projections = yield* ProjectionStore.ProjectionStoreV2;
        const now = yield* DateTime.now;
        const thread = makeThread(ThreadId.make("thread:large-stream-body"), now);
        const output = {
          content: [{ type: "text", text: "first line\n" + "x".repeat(9 * 1024 * 1024) }],
        };
        const tool: OrchestrationV2TurnItem = {
          id: TurnItemId.make("tool:large-stream-body"),
          type: "dynamic_tool",
          threadId: thread.id,
          runId: null,
          nodeId: null,
          providerThreadId: null,
          providerTurnId: null,
          nativeItemRef: null,
          parentItemId: null,
          ordinal: 1,
          status: "running",
          title: "Large tool",
          toolName: "mcp__large_tool",
          input: { query: "details" },
          output,
          startedAt: now,
          completedAt: null,
          updatedAt: now,
        };
        const [created] = yield* sink.write({
          events: [
            threadCreatedEvent({ id: "event:large-stream-body:created", thread, now }),
            {
              id: EventId.make("event:large-stream-body:replay"),
              type: "turn-item.updated",
              threadId: thread.id,
              occurredAt: now,
              payload: tool,
            },
          ],
        });
        const subscribe = () =>
          Stream.toPull(
            sink.stream({
              threadId: thread.id,
              afterSequence: created!.sequence,
              bounded: true,
            }),
          );
        const pull = yield* subscribe();
        const other = yield* subscribe();
        const replay = yield* pull;
        assert.lengthOf(replay, 1);
        assert.lengthOf(yield* other, 1);
        assert.equal(replay[0]!.event.type, "turn-item.updated");
        assert.isBelow(Buffer.byteLength(yield* encodeJson(replay)), 4_000);
        const stringify = yield* Effect.acquireRelease(
          Effect.sync(() => vi.spyOn(JSON, "stringify")),
          (spy) => Effect.sync(() => spy.mockRestore()),
        );
        const [completed] = yield* sink.write({
          events: [
            {
              id: EventId.make("event:large-stream-body:live"),
              type: "turn-item.updated",
              threadId: thread.id,
              occurredAt: now,
              payload: { ...tool, status: "completed", completedAt: now },
            },
          ],
        });
        const live = yield* pull;
        const otherLive = yield* other;
        assert.deepEqual(
          live.map((stored) => stored.sequence),
          [completed!.sequence],
        );
        // Both subscribers get one shared projection, so the live budget sizes it once.
        assert.strictEqual(otherLive[0], live[0]);
        assert.lengthOf(
          stringify.mock.calls.filter(([value]) => value === live[0] || value === otherLive[0]),
          1,
        );
        assert.isBelow(Buffer.byteLength(yield* encodeJson(live)), 4_000);
        for (const stored of [...replay, ...live]) {
          assert.equal(stored.event.type, "turn-item.updated");
          if (stored.event.type !== "turn-item.updated") return;
          assert.equal(stored.event.payload.type, "dynamic_tool");
          if (stored.event.payload.type !== "dynamic_tool") return;
          assert.notProperty(stored.event.payload, "output");
        }
        const persisted = yield* store.read({ threadId: thread.id }).pipe(Stream.runCollect);
        const fullEvent = persisted.find(
          (stored) => stored.sequence === completed!.sequence,
        )!.event;
        assert.equal(fullEvent.type, "turn-item.updated");
        if (fullEvent.type !== "turn-item.updated") return;
        assert.equal(fullEvent.payload.type, "dynamic_tool");
        if (fullEvent.payload.type !== "dynamic_tool") return;
        assert.deepEqual(fullEvent.payload.output, output);
        const detail = (yield* projections.getThreadProjection(thread.id)).turnItems.find(
          (item) => item.id === tool.id,
        )!;
        assert.equal(detail.type, "dynamic_tool");
        if (detail.type === "dynamic_tool") assert.deepEqual(detail.output, output);
      }),
    ),
  );

  it.effect.each(["high-water", "replay"] as const)(
    "bounds live events while the V2 %s query is blocked",
    (phase) =>
      Effect.scoped(
        Effect.gen(function* () {
          const sink = yield* EventSink.EventSinkV2;
          const now = yield* DateTime.now;
          const thread = makeThread(ThreadId.make(`thread:blocked-${phase}`), now);
          const [created] = yield* sink.write({
            events: [threadCreatedEvent({ id: `event:blocked-${phase}:created`, thread, now })],
          });
          const readStarted = yield* Deferred.make<void>();
          const readClosed = yield* Deferred.make<void>();
          const blockRead: Statement.Transformer = (statement) => {
            const [query] = statement.compile();
            if (
              query.includes("FROM orchestration_events") &&
              query.includes("MAX(sequence)") === (phase === "high-water")
            ) {
              return Deferred.succeed(readStarted, undefined).pipe(
                Effect.andThen(Effect.never),
                Effect.ensuring(Deferred.succeed(readClosed, undefined)),
                Effect.as(statement),
              );
            }
            return Effect.succeed(statement);
          };
          const reader = yield* sink
            .stream({ threadId: thread.id, afterSequence: created!.sequence, bounded: true })
            .pipe(
              Stream.provideService(Statement.CurrentTransformer, blockRead),
              Stream.runDrain,
              Effect.result,
              Effect.forkScoped,
            );
          yield* Deferred.await(readStarted);
          yield* sink.write({
            events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
              id: EventId.make(`event:blocked-${phase}:${index}`),
              type: "thread.metadata-updated" as const,
              threadId: thread.id,
              occurredAt: now,
              payload: { ...thread, title: `Updated ${index}` },
            })),
          });
          yield* Deferred.await(readClosed);
          const result = yield* Fiber.join(reader);
          assert.equal(result._tag, "Failure");
          if (result._tag === "Failure") {
            assert.equal(result.failure._tag, "EventSinkStreamError");
            assert.isTrue(isLiveStreamBufferError(result.failure.cause));
          }
        }),
      ),
  );

  it.effect(
    "keeps internal streams subscribed while replay is blocked beyond the RPC buffer cap",
    () =>
      Effect.scoped(
        Effect.gen(function* () {
          const sink = yield* EventSink.EventSinkV2;
          const now = yield* DateTime.now;
          const thread = makeThread(ThreadId.make("thread:internal-stream-burst"), now);
          const [created] = yield* sink.write({
            events: [
              threadCreatedEvent({ id: "event:internal-stream-burst:created", thread, now }),
            ],
          });
          const readStarted = yield* Deferred.make<void>();
          const releaseRead = yield* Deferred.make<void>();
          const blockHighWater: Statement.Transformer = (statement) => {
            const [query] = statement.compile();
            return query.includes("MAX(sequence)") && query.includes("FROM orchestration_events")
              ? Deferred.succeed(readStarted, undefined).pipe(
                  Effect.andThen(Deferred.await(releaseRead)),
                  Effect.as(statement),
                )
              : Effect.succeed(statement);
          };
          const eventCount = LIVE_STREAM_MAX_ITEMS + 1;
          const reader = yield* sink
            .stream({ threadId: thread.id, afterSequence: created!.sequence })
            .pipe(
              Stream.provideService(Statement.CurrentTransformer, blockHighWater),
              Stream.take(eventCount),
              Stream.runCollect,
              Effect.forkScoped,
            );
          yield* Deferred.await(readStarted);
          const written = yield* sink.write({
            events: Array.from({ length: eventCount }, (_, index) => ({
              id: EventId.make(`event:internal-stream-burst:${index}`),
              type: "thread.metadata-updated" as const,
              threadId: thread.id,
              occurredAt: now,
              payload: { ...thread, title: `Updated ${index}` },
            })),
          });
          yield* Deferred.succeed(releaseRead, undefined);
          assert.deepEqual(
            (yield* Fiber.join(reader)).map((event) => event.sequence),
            written.map((event) => event.sequence),
          );
        }),
      ),
  );

  it.effect("filters worker replay and live queues without losing matching events", () =>
    Effect.scoped(
      Effect.gen(function* () {
        const sink = yield* EventSink.EventSinkV2;
        const sql = yield* SqlClient.SqlClient;
        const now = yield* DateTime.now;
        const thread = makeThread(ThreadId.make("thread:filtered-worker"), now);
        const other = makeThread(ThreadId.make("thread:filtered-worker-other"), now);
        const run: OrchestrationV2Run = {
          id: RunId.make("run:filtered-worker"),
          threadId: thread.id,
          ordinal: 1,
          providerInstanceId,
          modelSelection,
          providerThreadId: null,
          userMessageId: MessageId.make("message:filtered-worker"),
          rootNodeId: null,
          activeAttemptId: null,
          status: "completed",
          queuePosition: null,
          requestedAt: now,
          startedAt: now,
          completedAt: now,
          checkpointId: null,
          contextHandoffId: null,
        };
        const runEvent = (id: string, payload: OrchestrationV2Run): OrchestrationV2DomainEvent => ({
          id: EventId.make(id),
          type: "run.updated",
          threadId: payload.threadId,
          runId: payload.id,
          occurredAt: now,
          payload,
        });
        const history = yield* sink.write({
          events: [
            threadCreatedEvent({ id: "event:filtered-worker:created", thread, now }),
            threadCreatedEvent({ id: "event:filtered-worker:other", thread: other, now }),
            runEvent("event:filtered-worker:history", run),
            runEvent("event:filtered-worker:other-history", {
              ...run,
              id: RunId.make("run:filtered-worker-other"),
              threadId: other.id,
            }),
          ],
        });
        const original = yield* sql<{ readonly payload_json: string }>`
          SELECT payload_json FROM orchestration_events
          WHERE sequence = ${history[0]!.sequence}
        `;
        yield* Effect.acquireRelease(
          sql`
            UPDATE orchestration_events SET payload_json = 'unreadable unrelated payload'
            WHERE sequence = ${history[0]!.sequence}
          `,
          () =>
            sql`
              UPDATE orchestration_events SET payload_json = ${original[0]!.payload_json}
              WHERE sequence = ${history[0]!.sequence}
            `.pipe(Effect.orDie),
        );
        const pull = yield* Stream.toPull(
          sink.stream({ threadId: thread.id, eventType: "run.updated" }),
        );
        assert.deepEqual(
          (yield* pull).map((stored) => stored.sequence),
          [history[2]!.sequence],
        );
        yield* sink.write({
          events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
            id: EventId.make(`event:filtered-worker:output:${index}`),
            type: "thread.metadata-updated" as const,
            threadId: thread.id,
            occurredAt: now,
            payload: { ...thread, title: `Output ${index}` },
          })),
        });
        const live = yield* sink.write({
          events: [
            runEvent("event:filtered-worker:live:1", { ...run, status: "interrupted" }),
            runEvent("event:filtered-worker:live:2", { ...run, status: "failed" }),
          ],
        });
        assert.deepEqual(
          (yield* pull).map((stored) => stored.sequence),
          live.map((stored) => stored.sequence),
        );
      }),
    ),
  );

  it.effect("paginates catch-up beyond the event-store read limit", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-large-catch-up");
      const thread = makeThread(threadId, now);
      const eventCount = 1_005;
      const events: Array<OrchestrationV2DomainEvent> = [
        threadCreatedEvent({ id: "event:foundation-catch-up:0", thread, now }),
        ...Array.from({ length: eventCount - 1 }, (_, index) => ({
          id: EventId.make(`event:foundation-catch-up:${index + 1}`),
          type: "thread.metadata-updated" as const,
          threadId,
          providerInstanceId,
          occurredAt: now,
          payload: {
            ...thread,
            title: `Catch-up update ${index + 1}`,
          },
        })),
      ];

      yield* eventSink.write({ events });
      const replayed = yield* eventSink.stream({ afterSequence: 0 }).pipe(
        Stream.take(eventCount),
        Stream.runCollect,
        Effect.map((events) => Array.from(events)),
      );

      assert.lengthOf(replayed, eventCount);
      assert.deepEqual(
        replayed.map((stored) => stored.sequence),
        Array.from({ length: eventCount }, (_, index) => index + 1),
      );
    }),
  );

  it.effect("does not lose or duplicate events while transitioning from catch-up to live", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const now = yield* DateTime.now;
      const threadId = ThreadId.make("thread:foundation-stream-race");
      const thread = makeThread(threadId, now);
      const created = yield* eventSink.write({
        events: [threadCreatedEvent({ id: "event:foundation-stream-race:0", thread, now })],
      });

      let afterSequence = created[0]!.sequence;
      // oxlint-disable-next-line t3code/no-polling-loop-in-tests -- Fixed-count writes exercise the catch-up/live handoff; each iteration awaits its event rather than polling.
      for (let index = 1; index <= 32; index += 1) {
        const nextEvent = {
          id: EventId.make(`event:foundation-stream-race:${index}`),
          type: "thread.metadata-updated" as const,
          threadId,
          providerInstanceId,
          occurredAt: now,
          payload: { ...thread, title: `Race update ${index}` },
        } satisfies OrchestrationV2DomainEvent;
        const reader = yield* eventSink
          .stream({ threadId, afterSequence })
          .pipe(Stream.runHead, Effect.forkChild);
        yield* Effect.yieldNow;
        const written = yield* eventSink.write({ events: [nextEvent] });
        const received = yield* Fiber.join(reader);
        if (Option.isNone(received)) {
          return yield* Effect.die("The event stream ended before delivering the live event.");
        }
        assert.equal(received.value.sequence, written[0]?.sequence);
        assert.equal(received.value.event.id, nextEvent.id);
        afterSequence = received.value.sequence;
      }
    }),
  );

  it.effect("replays shared provider-session payloads across every bound thread", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const maintenance = yield* ProjectionReplay.ProjectionReplay;
      const now = yield* DateTime.now;
      const firstThreadId = ThreadId.make("thread:foundation-shared-session:first");
      const secondThreadId = ThreadId.make("thread:foundation-shared-session:second");
      const providerSessionId = ProviderSessionId.make("provider-session:foundation:shared");
      const firstSession = {
        id: providerSessionId,
        driver: providerDriver,
        providerInstanceId,
        status: "ready" as const,
        cwd: "/workspace/first",
        model: modelSelection.model,
        capabilities: CodexProviderCapabilitiesV2,
        createdAt: now,
        updatedAt: now,
        lastError: null,
      };
      const secondSession = { ...firstSession, cwd: "/workspace/second" };

      yield* eventSink.write({
        events: [
          threadCreatedEvent({
            id: "event:foundation-shared-session:first-thread",
            thread: makeThread(firstThreadId, now),
            now,
          }),
          {
            id: EventId.make("event:foundation-shared-session:first-attachment"),
            type: "provider-session.attached",
            threadId: firstThreadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: firstSession,
          },
          threadCreatedEvent({
            id: "event:foundation-shared-session:second-thread",
            thread: makeThread(secondThreadId, now),
            now,
          }),
          {
            id: EventId.make("event:foundation-shared-session:second-attachment"),
            type: "provider-session.attached",
            threadId: secondThreadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: secondSession,
          },
        ],
      });

      assert.equal(
        (yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.cwd,
        secondSession.cwd,
      );
      assert.isTrue((yield* maintenance.verify).valid);
      assert.isTrue((yield* maintenance.rebuild).valid);
    }),
  );

  it.effect("prunes covered history, imported v1 events, and legacy receipts", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
      const sql = yield* SqlClient.SqlClient;
      const projections = yield* ProjectionStore.ProjectionStoreV2;
      const now = yield* DateTime.now;
      const nowIso = DateTime.formatIso(now);
      const threadId = ThreadId.make("thread:foundation-compact");
      const thread = makeThread(threadId, now);
      const messageId = MessageId.make("message:foundation-compact");
      const threadStateEvent = (
        suffix: string,
        type: "thread.visited" | "thread.metadata-updated" | "thread.tags-updated",
      ): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type,
        threadId,
        providerInstanceId,
        occurredAt: now,
        payload: { ...thread, lastVisitedAt: now },
      });
      const messageEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type: "message.updated",
        threadId,
        occurredAt: now,
        payload: {
          createdBy: "user",
          creationSource: "web",
          id: messageId,
          threadId,
          runId: null,
          nodeId: null,
          role: "user",
          text,
          attachments: [],
          streaming: false,
          createdAt: now,
          updatedAt: now,
        },
      });
      const nodeId = NodeId.make("node:foundation-compact");
      const nodeEvent = (
        suffix: string,
        status: "running" | "completed",
      ): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type: "node.updated",
        threadId,
        occurredAt: now,
        payload: {
          id: nodeId,
          threadId,
          runId: null,
          parentNodeId: null,
          rootNodeId: nodeId,
          kind: "assistant_message",
          status,
          countsForRun: false,
          providerThreadId: null,
          providerTurnId: null,
          nativeItemRef: null,
          runtimeRequestId: null,
          checkpointScopeId: null,
          startedAt: now,
          completedAt: status === "completed" ? now : null,
        },
      });
      const itemEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
        id: EventId.make(`event:foundation-compact:${suffix}`),
        type: "turn-item.updated",
        threadId,
        occurredAt: now,
        payload: {
          id: TurnItemId.make("turn-item:foundation-compact"),
          threadId,
          runId: null,
          nodeId: null,
          providerThreadId: null,
          providerTurnId: null,
          nativeItemRef: null,
          parentItemId: null,
          ordinal: 1,
          status: "completed",
          title: null,
          startedAt: now,
          completedAt: now,
          updatedAt: now,
          type: "assistant_message",
          messageId,
          text,
          streaming: false,
        },
      });

      yield* eventSink.write({
        events: [
          threadCreatedEvent({ id: "event:foundation-compact:create", thread, now }),
          threadStateEvent("meta", "thread.metadata-updated"),
          threadStateEvent("visit-1", "thread.visited"),
          messageEvent("message-1", "streaming"),
          nodeEvent("node-1", "running"),
          itemEvent("item-1", "streaming"),
          itemEvent("item-middle", "still streaming"),
          ...Array.from({ length: 501 }, (_, index) =>
            threadStateEvent(`history-${index}`, "thread.visited"),
          ),
          threadStateEvent("tags-1", "thread.tags-updated"),
          threadStateEvent("visit-2", "thread.visited"),
          messageEvent("message-2", "final"),
          nodeEvent("node-2", "completed"),
          itemEvent("item-2", "final"),
          // Older clients skip tag events, so these must not remove the rename above.
          threadStateEvent("tags-2", "thread.tags-updated"),
        ],
      });
      const beforeCompaction = yield* projections.getThreadProjection(threadId);

      const importedV1ThreadId = "thread:foundation-compact-v1-imported";
      const pendingV1ThreadId = "thread:foundation-compact-v1-pending";
      const insertV1Event = (threadIdValue: string, version: number) => sql`
        INSERT INTO orchestration_events (
          event_id, aggregate_kind, stream_id, stream_version, event_type,
          occurred_at, actor_kind, payload_json, metadata_json, application_event_version
        )
        VALUES (
          ${`event:v1:${threadIdValue}:${version}`}, 'thread', ${threadIdValue}, ${version},
          'thread.message-appended', ${nowIso}, 'user', '{}', '{}', 1
        )
      `;
      yield* insertV1Event(importedV1ThreadId, 1);
      yield* insertV1Event(importedV1ThreadId, 2);
      yield* insertV1Event(pendingV1ThreadId, 1);
      yield* sql`
        INSERT INTO orchestration_v2_legacy_imports (
          thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
          imported_message_count, last_error
        )
        VALUES
          (${importedV1ThreadId}, ${nowIso}, ${nowIso}, ${nowIso}, 2, NULL),
          (${pendingV1ThreadId}, ${nowIso}, ${nowIso}, NULL, 0, NULL)
        ON CONFLICT(thread_id) DO NOTHING
      `;
      yield* sql`
        INSERT INTO orchestration_command_receipts (
          command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
        )
        VALUES
          ('command:foundation-compact:legacy', 'thread', ${importedV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy'),
          ('command:foundation-compact:pending', 'thread', ${pendingV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy')
        ON CONFLICT(command_id) DO NOTHING
      `;

      // The first pass only marks the head and drops imported legacy rows.
      const marked = yield* maintenance.pruneEventStore();
      yield* TestClock.adjust("1 day");
      const summary = yield* maintenance.pruneEventStore();
      assert.isAtLeast(summary.deletedEventCount, 504);
      assert.equal(marked.deletedReceiptCount + summary.deletedReceiptCount, 1);

      const remaining = yield* sql<{ readonly event_id: string }>`
        SELECT event_id
        FROM orchestration_events
        WHERE aggregate_kind = 'thread'
          AND stream_id = ${threadId}
        ORDER BY sequence ASC
      `;
      assert.deepEqual(
        remaining.map((row) => row.event_id),
        ["event:foundation-compact:create", "event:foundation-compact:tags-2"],
      );

      const remainingV1 = yield* sql<{ readonly stream_id: string }>`
        SELECT stream_id
        FROM orchestration_events
        WHERE application_event_version = 1
          AND stream_id IN (${importedV1ThreadId}, ${pendingV1ThreadId})
      `;
      assert.deepEqual(
        remainingV1.map((row) => row.stream_id),
        [pendingV1ThreadId],
      );

      const remainingReceipts = yield* sql<{ readonly command_id: string }>`
        SELECT command_id
        FROM orchestration_command_receipts
        WHERE command_id IN ('command:foundation-compact:legacy', 'command:foundation-compact:pending')
      `;
      assert.deepEqual(
        remainingReceipts.map((row) => row.command_id),
        ["command:foundation-compact:pending"],
      );

      assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeCompaction);
    }),
  );

  it.effect("verifies and rebuilds projections with cross-thread subagent relations", () =>
    Effect.gen(function* () {
      const eventSink = yield* EventSink.EventSinkV2;
      const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
      const maintenance = yield* ProjectionReplay.ProjectionReplay;
      const sql = yield* SqlClient.SqlClient;
      const now = yield* DateTime.now;
      const parentThreadId = ThreadId.make("thread:foundation-cross-thread:parent");
      const childThreadId = ThreadId.make("thread:foundation-cross-thread:child");
      const childProviderThreadId = ProviderThreadId.make(
        "provider-thread:foundation-cross-thread:child",
      );
      const subagentId = NodeId.make("subagent:foundation-cross-thread");
      const spawnTransferId = ContextTransferId.make("transfer:foundation-cross-thread:spawn");
      const resultTransferId = ContextTransferId.make("transfer:foundation-cross-thread:result");
      const parentThread = makeThread(parentThreadId, now);
      const childThread = {
        ...makeThread(childThreadId, now),
        createdBy: "agent" as const,
        creationSource: "provider" as const,
        lineage: {
          parentThreadId,
          relationshipToParent: "subagent" as const,
          rootThreadId: parentThreadId,
        },
      };

      yield* eventSink.write({
        events: [
          threadCreatedEvent({
            id: "event:foundation-cross-thread:parent",
            thread: parentThread,
            now,
          }),
          threadCreatedEvent({
            id: "event:foundation-cross-thread:child",
            thread: childThread,
            now,
          }),
          {
            id: EventId.make("event:foundation-cross-thread:subagent"),
            type: "subagent.updated",
            threadId: parentThreadId,
            nodeId: subagentId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: subagentId,
              threadId: parentThreadId,
              runId: null,
              parentNodeId: NodeId.make("node:foundation-cross-thread:parent"),
              origin: "app_owned",
              createdBy: "agent",
              driver: providerDriver,
              providerInstanceId,
              providerThreadId: childProviderThreadId,
              childThreadId,
              nativeTaskRef: null,
              prompt: "Inspect the child flow",
              title: "Cross-thread child",
              model: modelSelection.model,
              status: "completed",
              result: "done",
              startedAt: now,
              completedAt: now,
              updatedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-cross-thread:spawn-transfer"),
            type: "context-transfer.created",
            threadId: childThreadId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: spawnTransferId,
              type: "subagent_spawn",
              sourceThreadId: parentThreadId,
              targetThreadId: childThreadId,
              sourcePoint: { threadId: parentThreadId },
              basePoint: null,
              sourceProviderInstanceId: providerInstanceId,
              targetProviderInstanceId: providerInstanceId,
              targetRunId: null,
              status: "consumed",
              resolution: null,
              createdBy: "agent",
              error: null,
              createdAt: now,
              updatedAt: now,
              consumedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-cross-thread:provider-thread"),
            type: "provider-thread.updated",
            threadId: parentThreadId,
            driver: providerDriver,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: childProviderThreadId,
              driver: providerDriver,
              providerInstanceId,
              providerSessionId: null,
              appThreadId: childThreadId,
              ownerNodeId: null,
              nativeThreadRef: null,
              nativeConversationHeadRef: null,
              status: "idle",
              firstRunOrdinal: 1,
              lastRunOrdinal: 1,
              handoffIds: [],
              forkedFrom: null,
              createdAt: now,
              updatedAt: now,
            },
          },
          {
            id: EventId.make("event:foundation-cross-thread:result-transfer"),
            type: "context-transfer.created",
            threadId: parentThreadId,
            providerInstanceId,
            occurredAt: now,
            payload: {
              id: resultTransferId,
              type: "subagent_result",
              sourceThreadId: childThreadId,
              targetThreadId: parentThreadId,
              sourcePoint: { threadId: childThreadId },
              basePoint: null,
              sourceProviderInstanceId: providerInstanceId,
              targetProviderInstanceId: providerInstanceId,
              targetRunId: null,
              status: "consumed",
              resolution: null,
              createdBy: "system",
              error: null,
              createdAt: now,
              updatedAt: now,
              consumedAt: now,
            },
          },
        ],
      });

      const assertCrossThreadProjection = Effect.gen(function* () {
        const parent = yield* projectionStore.getThreadProjection(parentThreadId);
        const child = yield* projectionStore.getThreadProjection(childThreadId);
        const expectedTransferIds = [spawnTransferId, resultTransferId].toSorted();
        assert.deepEqual(
          parent.contextTransfers.map((transfer) => transfer.id).toSorted(),
          expectedTransferIds,
        );
        assert.deepEqual(
          child.contextTransfers.map((transfer) => transfer.id).toSorted(),
          expectedTransferIds,
        );
        assert.deepEqual(
          parent.providerThreads.map((providerThread) => providerThread.id),
          [childProviderThreadId],
        );
        assert.equal(child.thread.activeProviderThreadId, childProviderThreadId);
      });

      yield* assertCrossThreadProjection;
      assert.isTrue((yield* maintenance.verify).valid);
      yield* sql`
        UPDATE orchestration_v2_projection_provider_threads SET payload_json = '{}'
        WHERE provider_thread_id = ${childProviderThreadId}
      `;
      assert.deepEqual(
        new Set((yield* maintenance.verify).unreadableThreadIds),
        new Set([parentThreadId, childThreadId]),
      );
      assert.isTrue((yield* maintenance.rebuild).valid);
      yield* assertCrossThreadProjection;
    }),
  );

  it.effect(
    "rolls back events, projections, receipts, and effects after a projection failure",
    () =>
      Effect.gen(function* () {
        const eventSink = yield* EventSink.EventSinkV2;
        const eventStore = yield* EventStore.EventStoreV2;
        const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
        const outbox = yield* EffectOutbox.EffectOutboxV2;
        const sql = yield* SqlClient.SqlClient;
        const now = yield* DateTime.now;
        const commandId = CommandId.make("command:foundation-atomic-failure");
        const threadId = ThreadId.make("thread:foundation-atomic-failure");
        const scopeId = CheckpointScopeId.make("scope:foundation-atomic-failure");
        const checkpoint = (index: number) => ({
          id: CheckpointId.make(`checkpoint:foundation-atomic-failure:${index}`),
          threadId,
          scopeId,
          runId: null,
          nodeId: NodeId.make("node:foundation-atomic-failure"),
          parentCheckpointId: null,
          ordinalWithinScope: 1,
          appRunOrdinal: null,
          ref: CheckpointRef.make(`checkpoint-ref:foundation-atomic-failure:${index}`),
          status: "ready" as const,
          files: [],
          capturedAt: now,
        });
        const events = [1, 2].map(
          (index) =>
            ({
              id: EventId.make(`event:foundation-atomic-failure:${index}`),
              type: "checkpoint.captured",
              threadId,
              occurredAt: now,
              payload: checkpoint(index),
            }) satisfies OrchestrationV2DomainEvent,
        );

        const exit = yield* Effect.exit(
          eventSink.commitCommand({
            commandId,
            threadId,
            commandType: "checkpoint.atomicity-test",
            acceptedAt: now,
            events,
            effects: [
              {
                id: "effect:foundation-atomic-failure",
                commandId,
                threadId,
                request: {
                  type: "provider-turn.start",
                  runId: RunId.make("run:foundation-atomic-failure"),
                },
              },
            ],
          }),
        );
        assert.equal(exit._tag, "Failure");
        assert.isTrue(Option.isNone(yield* receipts.getByCommandId(commandId)));
        assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
        assert.deepEqual(
          yield* eventStore.readByCommandId({ commandId }).pipe(
            Stream.runCollect,
            Effect.map((events) => Array.from(events)),
          ),
          [],
        );
        const checkpointRows = yield* sql<{ readonly count: number }>`
        SELECT COUNT(*) AS count
        FROM orchestration_v2_projection_checkpoints
        WHERE thread_id = ${threadId}
      `;
        assert.equal(checkpointRows[0]?.count, 0);
      }),
  );

  it.effect("replays every command event across bounded persistence pages", () =>
    Effect.gen(function* () {
      const sql = yield* SqlClient.SqlClient;
      const eventStore = yield* EventStore.EventStoreV2;
      const eventSink = yield* EventSink.EventSinkV2;
      const now = yield* DateTime.now;
      const occurredAt = DateTime.formatIso(now);
      const commandId = CommandId.make("command:paged-command-replay");
      const exactPageCommandId = CommandId.make("command:paged-command-replay:exact");
      const emptyCommandId = CommandId.make("command:paged-command-replay:empty");
      const threadId = "thread:paged-command-replay";

      const eventRow = (
        ordinal: number,
        input: {
          readonly commandId?: string | null;
          readonly aggregateKind?: "project" | "thread";
          readonly streamId?: string;
          readonly version?: number;
        } = {},
      ) => ({
        event_id: `event:paged-command-replay:${ordinal}`,
        aggregate_kind: input.aggregateKind ?? "thread",
        stream_id: input.streamId ?? threadId,
        stream_version: ordinal,
        event_type: "provider-session.detached",
        occurred_at: occurredAt,
        command_id: input.commandId ?? null,
        causation_event_id: null,
        correlation_id: null,
        actor_kind: "server",
        payload_json: JSON.stringify({
          providerSessionId: `session:paged-command-replay:${ordinal}`,
          detachedAt: occurredAt,
        }),
        metadata_json: "{}",
        application_event_version: input.version ?? 2,
      });

      const matchingCount = 1_001;