sailfork / t3code-github / apps/ server/ src/ orchestration-v2/ Orchestrator.ts

import type {
  GuestShareId,
  OrchestrationV2SearchThreadInput,
  OrchestrationV2SearchThreadResult,
  OrchestrationV2ThreadHistoryPage,
} from "@t3tools/contracts";
import {
  latestExecutedRun,
  latestRootProviderFailure,
  runRanAfter,
  usageLimitBlockedRun,
} from "@t3tools/shared/orchestrationV2ThreadError";
import { canonicalizeGitCafePullRequestUrl } from "@t3tools/source-control-definitions/registry";
import { threadPullRequestsOf } from "../sourceControl/threadPullRequests.ts";
import {
  normalizeThreadPullRequestKey,
  visibleThreadPullRequests,
  threadPullRequestKeysEqual,
  legacyThreadPullRequestKey,
} from "../sourceControl/threadPullRequests.ts";
import {
  ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE,
  type ChatAttachment,
  CommandId,
  type GuestMessageId,
  isCompactionCommandMessage,
  isOrchestrationV2WorkActive,
  isProviderNativeSubagentThread,
  MessageId,
  type NodeId,
  type ModelSelection,
  OrchestrationV2Command,
  type OrchestrationV2InternalCommand,
  type OrchestrationV2LimitRecoveryUpdate,
  type OrchestrationV2ServerCommand,
  type ThreadPullRequestLink,
  type ThreadPullRequestWatch,
  type OrchestrationV2AppThread,
  type OrchestrationV2ContextHandoff,
  type OrchestrationV2ContextSourcePoint,
  type OrchestrationV2ContextTransfer,
  type OrchestrationV2ContextTransferResolution,
  type OrchestrationV2ConversationMessage,
  type OrchestrationV2DelegatedCompletionCohort,
  type OrchestrationV2DelegatedCompletionDelivery,
  type OrchestrationV2DomainEvent,
  type OrchestrationV2ExecutionNode,
  type OrchestrationV2ProviderSession,
  type OrchestrationV2ProviderThread,
  type OrchestrationV2ProviderTurn,
  type OrchestrationV2RemoteTaskChild,
  type OrchestrationV2ModelSwitch,
  type OrchestrationV2Run,
  type OrchestrationV2RunAttempt,
  type OrchestrationV2ThreadShell,
  type OrchestrationV2ThreadShellSnapshot,
  type OrchestrationV2StoredEvent,
  type OrchestrationV2Subagent,
  type OrchestrationV2ThreadProjection,
  type OrchestrationV2TurnItem,
  latestProviderTurnForAttempt,
  parseGoalCommand,
  orchestrationV2RunWorkStartedAt,
  orchestrationV2WorktreePending,
  ProviderInstanceId,
  ProviderInteractionMode,
  isRuntimeModeAbove,
  resolveProviderInstanceEnabled,
  resolveProjectProviderInstanceEnabled,
  type ProviderSessionId,
  RunId,
  RuntimeMode,
  ThreadLinkedPullRequest,
  ThreadId,
  TurnItemId,
  TrimmedNonEmptyString,
  managedRuntimeModeCeilingMessage,
} from "@t3tools/contracts";
import { modelSelectionsEqual } from "@t3tools/shared/model";
import { modelSwitchSince } from "./modelSwitch.ts";
import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
import {
  backgroundWorkHoldsCompletion,
  derivePendingBackgroundWork,
  pendingBackgroundTurnItems,
} from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Ref from "effect/Ref";
import * as Result from "effect/Result";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";

import * as ProjectStore from "./ProjectStore.ts";
import {
  isCheckpointRestoreIsolated,
  SHARED_WORKSPACE_RESTORE_MESSAGE,
} from "./CheckpointRestoreSafety.ts";
import { CheckpointServiceV2 } from "./CheckpointService.ts";
import {
  CommandPolicyV2,
  goalCommandStopsActiveRun,
  isActiveRun,
  isNativeMaintenanceCommand,
  resolveMessageDispatchIntent,
} from "./CommandPolicy.ts";
import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts";
import {
  ContextHandoffServiceV2,
  readForkHistoryItems,
  readInheritedHandoffItems,
  visibleConversationItems,
  visibleDeltaRunOrdinals,
} from "./ContextHandoffService.ts";
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
import {
  cancelledBeforeStart,
  continuationFailedBeforePrompt,
  continuationStartRetryDelayMs,
  isRestartContinuationSource,
  runOutcome,
} from "./RestartBackgroundNote.ts";
import {
  isBufferedProviderNotification,
  isUndeliveredMailboxSteer,
} from "./NotificationMailbox.ts";
import { EventSinkV2 } from "./EventSink.ts";
import {
  decideGuestMessageSend,
  guestShareAfterCancelledSend,
  releasedGuestAttachmentIds,
  withoutGuestAttribution,
  decideGuestShareCommand,
  type GuestShareCommand,
} from "./GuestShareDecider.ts";
import * as EffectOutbox from "./EffectOutbox.ts";
import { withCommandMetrics } from "../observability/Metrics.ts";
import type { OrchestrationEffectRequestV2, PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
import {
  DispatchModeLimit,
  exceededDispatchModeLimit,
  narrowToCeiling,
  type RuntimeModeCeiling,
  runtimeModeCeilingRefusal,
  threadsStartingWork,
} from "./DispatchModeLimit.ts";
import {
  applyToProjection,
  emptyProjection,
  threadShellFromProjection,
  ProjectionStoreV2,
  type ProjectionRecordField,
  type ProjectionTimelinePage,
  type ProjectionTimelinePageOptions,
  type ProjectionRecordFilter,
  type ProjectionRecords,
  type ProjectionCheckpointContext,
  type OlderSettledThreadsListing,
  type ActiveShellIndexEntry,
  type OlderSettledThreadsQuery,
  type ShellSnapshotOptions,
} from "./ProjectionStore.ts";
import { ProviderAdapterRegistryV2 } from "./ProviderAdapterRegistry.ts";
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
import * as RunExecutionService from "./RunExecutionService.ts";
import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts";
import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts";
import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
import { isRuntimePolicyAboveCeilingError, RuntimePolicyV2 } from "./RuntimePolicy.ts";
import {
  endOrphanedNativeSubagent,
  endRunlessRootTurns,
  makeSubagentChildThread,
  subagentResultForRun,
  delegatedTaskProgress,
  subagentThreadTitle,
} from "@t3tools/provider-core/server/subagentProjection";
import {
  forkableSourceRunStatusError,
  isForkableSourceRunStatus,
  ThreadForkServiceV2,
} from "./ThreadForkService.ts";
import { planThreadDeletion } from "./ThreadDeletion.ts";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import * as ServerSettings from "../serverSettings.ts";

const isMessageQuestionAnswer = Schema.is(
  Schema.Union([TrimmedNonEmptyString, Schema.NonEmptyArray(TrimmedNonEmptyString)]),
);

export class OrchestratorDispatchError extends Schema.TaggedError<OrchestratorDispatchError>()(
  "OrchestratorDispatchError",
  {
    commandId: CommandId,
    commandType: Schema.String,
    cause: Schema.optional(Schema.Defect()),
  },
) {
  override get message(): string {
    return `Failed to dispatch orchestration command ${this.commandType} (${this.commandId}).`;
  }
}

export class OrchestratorThreadAlreadyExistsError extends Schema.TaggedError<OrchestratorThreadAlreadyExistsError>()(
  "OrchestratorThreadAlreadyExistsError",
  { commandId: CommandId, threadId: ThreadId },
) {
  override get message(): string {
    return `Thread ${this.threadId} already exists. Use a new thread id.`;
  }
}

export class OrchestratorCommandRejectedError extends Schema.TaggedError<OrchestratorCommandRejectedError>()(
  "OrchestratorCommandRejectedError",
  { commandId: CommandId, commandType: Schema.String, cause: Schema.optional(Schema.Defect()) },
) {
  override get message(): string {
    return `Orchestration command ${this.commandType} (${this.commandId}) was rejected before commit.`;
  }
}

export class OrchestratorProjectionError extends Schema.TaggedError<OrchestratorProjectionError>()(
  "OrchestratorProjectionError",
  {
    threadId: ThreadId,
    cause: Schema.optional(Schema.Defect()),
  },
) {
  override get message(): string {
    return `Failed to load orchestration projection for thread ${this.threadId}.`;
  }
}

const isOrchestratorProjectionError = Schema.is(OrchestratorProjectionError);

export class OrchestratorDomainEventStreamError extends Schema.TaggedError<OrchestratorDomainEventStreamError>()(
  "OrchestratorDomainEventStreamError",
  {
    cause: Schema.optional(Schema.Defect()),
  },
) {
  override get message(): string {
    return "Failed while streaming orchestration domain events.";
  }
}

export class OrchestratorProviderAdapterError extends Schema.TaggedError<OrchestratorProviderAdapterError>()(
  "OrchestratorProviderAdapterError",
  {
    commandId: CommandId,
    providerInstanceId: ProviderInstanceId,
    cause: Schema.optional(Schema.Defect()),
  },
) {
  override get message(): string {
    return `Provider adapter failed while dispatching orchestration command ${this.commandId}.`;
  }
}

export class OrchestratorSubagentThreadReadOnlyError extends Schema.TaggedError<OrchestratorSubagentThreadReadOnlyError>()(
  "OrchestratorSubagentThreadReadOnlyError",
  { commandId: CommandId, threadId: ThreadId },
) {
  override get message(): string {
    return "This subagent is run by its provider and cannot take messages. Message the parent thread instead.";
  }
}

export class OrchestratorThreadAboveModeLimitError extends Schema.TaggedError<OrchestratorThreadAboveModeLimitError>()(
  "OrchestratorThreadAboveModeLimitError",
  {
    commandId: CommandId,
    threadId: ThreadId,
    mode: Schema.Literals(["runtime", "interaction"]),
    runtimeMode: RuntimeMode,
    interactionMode: ProviderInteractionMode,
  },
) {
  override get message(): string {
    return `Thread ${this.threadId} now runs in ${this.runtimeMode}/${this.interactionMode} mode, above what this caller may change.`;
  }
}

const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError);

/** The ceiling refusal under however many dispatch errors wrapped it on the way out. */
function runtimePolicyCeilingCause(error: unknown) {
  let current = error;
  while (isOrchestratorDispatchError(current)) current = current.cause;
  return isRuntimePolicyAboveCeilingError(current) ? current : undefined;
}

export class OrchestratorRuntimeModeAboveCeilingError extends Schema.TaggedError<OrchestratorRuntimeModeAboveCeilingError>()(
  "OrchestratorRuntimeModeAboveCeilingError",
  {
    commandId: CommandId,
    threadId: ThreadId,
    refused: Schema.Literals(["mode", "turn"]),
    runtimeMode: RuntimeMode,
    maxRuntimeMode: RuntimeMode,
    organization: Schema.String,
  },
) {
  override get message(): string {
    const limit = managedRuntimeModeCeilingMessage(this.organization, this.maxRuntimeMode);
    return this.refused === "turn" ? `${limit} Switch this thread's mode to continue.` : limit;
  }
}

export class OrchestratorCommandPreviouslyRejectedError extends Schema.TaggedError<OrchestratorCommandPreviouslyRejectedError>()(
  "OrchestratorCommandPreviouslyRejectedError",
  {
    commandId: CommandId,
    commandType: Schema.String,
    detail: Schema.String,
  },
) {
  override get message(): string {
    return `Command ${this.commandId} was previously rejected: ${this.detail}`;
  }
}

export class OrchestratorCommandIdConflictError extends Schema.TaggedError<OrchestratorCommandIdConflictError>()(
  "OrchestratorCommandIdConflictError",
  {
    commandId: CommandId,
    commandType: Schema.String,
    receiptThreadId: ThreadId,
    commandThreadId: ThreadId,
  },
) {
  override get message(): string {
    return `Command ${this.commandId} was already handled for thread ${this.receiptThreadId} and cannot be replayed for ${this.commandThreadId}.`;
  }
}

export const OrchestratorV2Error = Schema.Union([
  OrchestratorDispatchError,
  OrchestratorThreadAlreadyExistsError,
  OrchestratorCommandRejectedError,
  OrchestratorProjectionError,
  OrchestratorDomainEventStreamError,
  OrchestratorProviderAdapterError,
  OrchestratorCommandPreviouslyRejectedError,
  OrchestratorCommandIdConflictError,
  OrchestratorSubagentThreadReadOnlyError,
  OrchestratorThreadAboveModeLimitError,
  OrchestratorRuntimeModeAboveCeilingError,
]);
export type OrchestratorV2Error = typeof OrchestratorV2Error.Type;

export interface OrchestratorV2DispatchResult {
  readonly sequence: number;
  readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
  /** The command id was already accepted; analytics skip it. */
  readonly replayed?: true;
}

export interface OrchestratorV2Shape {
  readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
  readonly recoverDelegatedTasks: Effect.Effect<void>;
  readonly recoverDelegatedTask: (threadId: ThreadId, sourceRunId: RunId) => Effect.Effect<void>;
  readonly delegatedTaskResultPending: (
    childThreadId: ThreadId,
    taskId?: NodeId,
  ) => Effect.Effect<boolean, OrchestratorProjectionError>;
  readonly restartContinuationPending: (run: OrchestrationV2Run) => Effect.Effect<boolean>;
  /** Declined continuations emit no thread event; this still wakes waiters. */
  readonly awaitRestartContinuation: (run: OrchestrationV2Run) => Effect.Effect<void>;
  readonly dispatch: (
    command: OrchestrationV2ServerCommand,
  ) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
  readonly autoArmLimitRecovery: (input: {
    readonly commandId: CommandId;
    readonly threadId: ThreadId;
    readonly limitRecovery: OrchestrationV2LimitRecoveryUpdate;
  }) => Effect.Effect<void, OrchestratorV2Error>;
  readonly searchThreadStream: (
    input: OrchestrationV2SearchThreadInput,
  ) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
  readonly searchThread: (
    input: OrchestrationV2SearchThreadInput,
  ) => Effect.Effect<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
  readonly getThreadHistoryPage: (
    threadId: ThreadId,
    cursor: string,
    throughEntryId?: string,
    conversationOnly?: boolean,
    workMode?: "deferred" | "work",
  ) => Effect.Effect<OrchestrationV2ThreadHistoryPage, OrchestratorV2Error>;
  readonly getTimelinePage: (
    threadId: ThreadId,
    options: ProjectionTimelinePageOptions,
  ) => Effect.Effect<ProjectionTimelinePage, OrchestratorProjectionError>;
  readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, OrchestratorV2Error>;
  readonly getTurnItem: (input: {
    readonly threadId: ThreadId;
    readonly itemId: TurnItemId;
  }) => Effect.Effect<OrchestrationV2TurnItem | null, OrchestratorV2Error>;
  readonly getThreadRecords: <K extends ProjectionRecordField>(
    threadId: ThreadId,
    fields: ReadonlyArray<K>,
    filter?: ProjectionRecordFilter,
  ) => Effect.Effect<ProjectionRecords<K>, OrchestratorV2Error>;
  readonly listAppOwnedChildThreadIds: (
    threadId: ThreadId,
  ) => Effect.Effect<ReadonlyArray<ThreadId>, OrchestratorV2Error>;
  readonly getThreadProjection: (
    threadId: ThreadId,
  ) => Effect.Effect<OrchestrationV2ThreadProjection, OrchestratorV2Error>;
  readonly getCheckpointContext: (
    threadId: ThreadId,
  ) => Effect.Effect<ProjectionCheckpointContext, OrchestratorV2Error>;
  readonly getThreadSnapshot: (threadId: ThreadId) => Effect.Effect<
    {
      readonly schemaVersion: number;
      readonly snapshotSequence: number;
      readonly projection: OrchestrationV2ThreadProjection;
    },
    OrchestratorV2Error
  >;
  readonly getThreadSnapshotWindow: (
    threadId: ThreadId,
    options: Parameters<ProjectionStoreV2["Service"]["getThreadSnapshotWindow"]>[1],
  ) => Effect.Effect<
    {
      readonly schemaVersion: number;
      readonly snapshotSequence: number;
      readonly projection: OrchestrationV2ThreadProjection;
    },
    OrchestratorV2Error
  >;
  readonly getPullRequestSnapshot: (
    options: Parameters<ProjectionStoreV2["Service"]["getPullRequestSnapshot"]>[0],
  ) => Effect.Effect<
    Effect.Success<ReturnType<ProjectionStoreV2["Service"]["getPullRequestSnapshot"]>>,
    OrchestratorV2Error
  >;
  readonly getAgentAwarenessSnapshot: (
    threadId?: ThreadId,
  ) => Effect.Effect<
    Effect.Success<ReturnType<ProjectionStoreV2["Service"]["getAgentAwarenessSnapshot"]>>,
    OrchestratorV2Error
  >;
  readonly getShellSnapshot: (
    options?: ShellSnapshotOptions,
  ) => Effect.Effect<OrchestrationV2ThreadShellSnapshot, OrchestratorV2Error>;
  readonly listActiveSubagentThreadIds: () => Effect.Effect<
    ReadonlyArray<ThreadId>,
    OrchestratorV2Error
  >;
  readonly countThreads: () => Effect.Effect<number, OrchestratorV2Error>;
  readonly listActiveShellIndex: () => Effect.Effect<
    ReadonlyArray<ActiveShellIndexEntry>,
    OrchestratorV2Error
  >;
  readonly listOlderSettledThreads: (
    query: OlderSettledThreadsQuery,
  ) => Effect.Effect<OlderSettledThreadsListing, OrchestratorV2Error>;
  readonly readShellSnapshot: (
    options?: ShellSnapshotOptions,
  ) => Effect.Effect<
    Effect.Effect<OrchestrationV2ThreadShellSnapshot, OrchestratorV2Error>,
    OrchestratorV2Error
  >;
  readonly getThreadShell: (
    threadId: ThreadId,
  ) => Effect.Effect<OrchestrationV2ThreadShell | null, OrchestratorV2Error>;
  readonly findGuestShareThreadId: (
    shareId: GuestShareId,
  ) => Effect.Effect<ThreadId | null, OrchestratorV2Error>;
  /** Titles of the given threads that exist, archived ones included. */
  readonly getThreadTitles: (
    threadIds: ReadonlyArray<ThreadId>,
  ) => Effect.Effect<Record<ThreadId, string>, OrchestratorV2Error>;
  readonly getThreadEventSequence: (
    threadId: ThreadId,
  ) => Effect.Effect<number, OrchestratorV2Error>;
  readonly streamStoredEvents: Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
  /** Events arrive projected for the wire and shared by every subscriber; never mutate them. */
  readonly streamStoredEventsFrom: (input?: {
    readonly threadId?: ThreadId;
    readonly afterSequence?: number;
    readonly eventType?: OrchestrationV2DomainEvent["type"];
  }) => Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
  readonly streamDomainEvents: Stream.Stream<OrchestrationV2DomainEvent, OrchestratorV2Error>;
}

export class OrchestratorV2 extends Context.Service<OrchestratorV2, OrchestratorV2Shape>()(
  "t3/orchestration-v2/Orchestrator/OrchestratorV2",
) {}

/** Run ordinals can skip values: a queued run moves past newer runs when it starts. */
function nextRunOrdinal(projection: Pick<OrchestrationV2ThreadProjection, "runs">): number {
  return projection.runs.reduce((max, run) => Math.max(max, run.ordinal), 0) + 1;
}

/** Stamp at wake start; queued prompts lack start times and delegated results jump the queue. */
function wakeWorkStartedAt(
  runs: ReadonlyArray<OrchestrationV2Run>,
  trigger: {
    readonly notification?: unknown;
    readonly delegatedCompletion?: unknown;
    readonly restartContinuationOfRunId?: RunId | undefined;
  },
): Pick<OrchestrationV2Run, "workStartedAt"> {
  if (
    trigger.notification === undefined &&
    trigger.delegatedCompletion === undefined &&
    trigger.restartContinuationOfRunId === undefined
  ) {
    return {};
  }
  const previous = runs
    .flatMap((run) => (run.startedAt === null ? [] : [{ run, startedAt: run.startedAt }]))
    .toSorted(
      (left, right) =>
        DateTime.Order(right.startedAt, left.startedAt) || right.run.ordinal - left.run.ordinal,
    )[0]?.run;
  return previous === undefined ? {} : { workStartedAt: orchestrationV2RunWorkStartedAt(previous) };
}

const threadPullRequestLinksEqual = Schema.toEquivalence(Schema.NullOr(ThreadLinkedPullRequest));

/** Unfinished runs (including queued) or open requests block moving a thread to another project. */
export function threadHasUnfinishedWork(records: {
  readonly runs: ReadonlyArray<{ readonly status: string }>;
  readonly runtimeRequests: ReadonlyArray<{ readonly status: string }>;
}): boolean {
  return (
    records.runs.some(
      (run) =>
        run.status !== "completed" &&
        run.status !== "failed" &&
        run.status !== "cancelled" &&
        run.status !== "interrupted" &&
        run.status !== "rolled_back",
    ) || records.runtimeRequests.some((request) => request.status === "pending")
  );
}

function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
  switch (command.type) {
    case "thread.create":
    case "thread.archive":
    case "thread.unarchive":
    case "thread.delete":
    case "thread.guest-share.open":
    case "thread.guest-share.redeem":
    case "thread.guest-share.close":
    case "thread.guest-message.submit":
    case "thread.guest-message.dismiss":
    case "thread.settle":
    case "thread.auto-settle":
    case "thread.unsettle":
    case "thread.snooze":
    case "thread.unsnooze":
    case "thread.auto-settle.set":
    case "thread.pin":
    case "thread.unpin":
    case "thread.pin.reorder":
    case "thread.active.reorder":
    case "thread.tag.set":
    case "thread.visit":
    case "thread.mark-unread":
    case "thread.metadata.update":
    case "thread.pull-request.link":
    case "thread.pull-request.unlink":
    case "thread.pull-request-link.sync":
    case "thread.pull-request.watch":
    case "thread.pull-request-watch.sync":
    case "thread.pull-request.sync":
    case "thread.title.regeneration.complete":
    case "thread.project.adopt-folder":
    case "thread.runtime-mode.set":
    case "thread.interaction-mode.set":
    case "thread.model-selection.set":
    case "provider-session.detach":
    case "message.dispatch":
    case "notification.delivery.accept":
    case "prepared-run.release":
    case "prepared-run.progress":
    case "prepared-run.fail":
    case "prepared-run.retry":
    case "subagent.stop":
    case "run.interrupt":
    case "queued-message.promote-to-steer":
    case "queued-run.start":
    case "queue.resume":
    case "queued-run.reorder":
    case "queued-run.cancel":
    case "queued-run.edit":
    case "runtime-request.respond":
    case "runtime-request.delivery.fail":
    case "thread.user-input.dismiss":
    case "checkpoint.rollback":
    case "checkpoint.rollback.fail":
    case "thread.background-work.settle":
    case "thread.interrupt.unresponsive":
    case "thread.stop":
    case "provider.switch":
      return command.threadId;
    case "delegated_task.request":
    case "delegated_task.remote.request":
    case "delegated_task.remote.complete":
    case "delegated_task.remote.stop":
    case "delegated_task.wake-policy":
    case "delegated_task.completion-delivery.acknowledge":
    case "delegated_task.completion-delivery.dispose":
    case "thread.created.record":
      return command.parentThreadId;
    case "secret_request.record":
    case "link_request.record":
      return command.threadId;
    case "thread.fork":
    case "thread.merge_back":
      return command.targetThreadId;
  }
}

// Validate before resolving the request: adapters may otherwise coerce unusable answers.
const decodeUserInputAnswers = Schema.decodeUnknownEffect(
  Schema.Record(
    Schema.String,
    Schema.Union([
      Schema.String,
      Schema.Array(Schema.String),
      Schema.Struct({ answers: Schema.Array(Schema.String) }),
    ]),
  ),
);

function pendingThreadTitleGenerationEffect(
  commandId: CommandId,
  threadId: ThreadId,
  kind:
    | { readonly type: "initial"; readonly messageId: MessageId }
    | { readonly type: "regenerate" },
): PendingOrchestrationEffectV2 {
  return {
    id: `effect:${commandId}:thread-title.generate`,
    commandId,
    threadId,
    request: { type: "thread-title.generate", kind },
  };
}

function delegatedTasksStopEffect(command: {
  readonly commandId: CommandId;
  readonly threadId: ThreadId;
  readonly reason?: string | undefined;
}): PendingOrchestrationEffectV2 {
  return {
    id: `effect:${command.commandId}:delegated-tasks.stop`,
    commandId: command.commandId,
    threadId: command.threadId,
    request: {
      type: "delegated-tasks.stop",
      ...(command.reason === undefined ? {} : { reason: command.reason }),
    },
  };
}

const WORKSPACE_PREPARATION_INPUT = "Preparing workspace";
const RUNTIME_REQUEST_DELIVERY_FAILED_MESSAGE =
  "T3 Code could not confirm that your answer reached the agent. If the run stays stuck, stop it and try again.";

function withoutPreparationResult(
  item: Extract<OrchestrationV2TurnItem, { readonly type: "command_execution" }>,
) {
  const { output: _output, exitCode: _exitCode, outputIndicatesFailure: _failure, ...rest } = item;
  return rest;
}

/** Stop ended this task, so its result is not delivered to the parent agent. */
function cancelledAppOwnedTask(
  task: OrchestrationV2Subagent,
  now: DateTime.Utc,
): OrchestrationV2Subagent {
  return {
    ...task,
    status: "cancelled",
    completedAt: now,
    updatedAt: now,
    completionDelivery: { state: "disposed", observedByRunId: null },
  };
}

// Provider wakes can start follow-up work; monitor updates only report progress.
// Dispatch takes the parent's lock for follow-ups because intake writes the parent.
function mayStartSubagentFollowup(
  command: Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>,
): boolean {
  if (command.scheduledTaskId !== undefined || command.restartContinuationOfRunId !== undefined)
    return false;
  if (command.notification !== undefined) return command.notification.source.kind !== "monitor";
  return (
    command.createdBy === "user" ||
    command.creationSource === "mcp" ||
    command.usageLimitContinuationOfRunId !== undefined
  );
}

/** Waiting runs only drain post-terminal output; their agent turn is over and still needs a wake. */
function isLiveRun(run: OrchestrationV2Run): boolean {
  return run.status === "preparing" || run.status === "starting" || run.status === "running";
}

function hasLiveRun(projection: Pick<OrchestrationV2ThreadProjection, "runs">): boolean {
  return projection.runs.some(isLiveRun);
}

function withPullRequestWatch(
  link: ThreadPullRequestLink,
  watch: ThreadPullRequestWatch | undefined,
): ThreadPullRequestLink {
  const { watch: previous, endedWatch, ...rest } = link;
  if (watch !== undefined) return { ...rest, watch };
  const ended = previous ?? endedWatch;
  return ended === undefined ? rest : { ...rest, endedWatch: ended };
}

function legacyPullRequestLink(
  thread: OrchestrationV2AppThread,
  linked: ThreadLinkedPullRequest,
  now: DateTime.Utc,
): ThreadPullRequestLink {
  const key = legacyThreadPullRequestKey(linked);
  const existing = threadPullRequestsOf(thread).find((link) =>
    threadPullRequestKeysEqual(link, key),
  );
  return withPullRequestWatch(
    {
      ...key,
      url: canonicalizeGitCafePullRequestUrl(linked.url),
      source: "manual",
      linkedAt: DateTime.formatIso(now),
      snapshot: null,
      stack: null,
      ...(existing?.repositoryWebUrl !== undefined && {
        repositoryWebUrl: existing.repositoryWebUrl,
      }),
      ...(existing?.endedWatch && { endedWatch: existing.endedWatch }),
    },
    existing?.watch,
  );
}

const DELEGATED_RESULT_INLINE_BYTES = 16_000;

function delegatedCompletionWakeDetail(
  taskIds: ReadonlyArray<string>,
  tasks: ReadonlyArray<OrchestrationV2Subagent>,
): string {
  let remainingBytes = DELEGATED_RESULT_INLINE_BYTES;
  return taskIds
    .map((taskId) => {
      const task = tasks.find((candidate) => candidate.id === taskId);
      const label = `Delegated task ${taskId}${task?.title == null ? "" : ` (${task.title})`}${task === undefined ? " reached a terminal state" : ` [${task.status}]`}`;
      if (task?.result != null) {
        const detail = `${label}:\n${task.result}`;
        const bytes = Buffer.byteLength(detail);
        if (bytes <= remainingBytes) {
          remainingBytes -= bytes;
          return detail;
        }
      }
      return `${label}. Use task_status with taskId ${taskId} to read the result.`;
    })
    .join("\n\n");
}

function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean {
  return (
    status === "completed" ||
    status === "failed" ||
    status === "cancelled" ||
    status === "interrupted"
  );
}

function delegatedTaskTerminalStatus(
  status: OrchestrationV2Run["status"],
): OrchestrationV2Subagent["status"] | null {
  switch (status) {
    case "completed":
    case "failed":
    case "cancelled":
    case "interrupted":
      return status;
    case "rolled_back":
      return "cancelled";
    case "preparing":
    case "queued":
    case "starting":
    case "running":
    case "waiting":
      return null;
  }
}

/** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */
function worktreeSetupPending(
  projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
): boolean {
  return (
    projection.thread.worktreePath === null &&
    projection.runs.some(
      (run) =>
        (run.status === "failed" || run.status === "interrupted") &&
        run.workspacePreparation?.type === "worktree",
    )
  );
}

function nextQueuedRun(
  projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
): OrchestrationV2Run | undefined {
  return queuedRunsInDeliveryOrder(projection).find((run) => run.queueHeld !== true);
}

function followsCurrentConversation(
  message: OrchestrationV2ConversationMessage | undefined,
): boolean {
  return (
    message?.delegatedCompletion !== undefined ||
    (message?.notification?.source.kind === "monitor" && !isBufferedProviderNotification(message))
  );
}

function queuedRunModelSelection(
  projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
  run: OrchestrationV2Run,
): ModelSelection {
  const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
  return followsCurrentConversation(message)
    ? projection.thread.modelSelection
    : run.modelSelection;
}

function usageLimitHoldsQueue(
  projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
  providerInstanceId: ModelSelection["instanceId"],
): boolean {
  let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
  for (const session of projection.providerSessions) {
    if (
      session.providerInstanceId === providerInstanceId &&
      (latestSession === undefined ||
        DateTime.isGreaterThan(session.updatedAt, latestSession.updatedAt))
    ) {
      latestSession = session;
    }
  }
  return (
    usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
      ?.providerInstanceId === providerInstanceId
  );
}

function latestStableRun(
  projection: Pick<OrchestrationV2ThreadProjection, "runs">,
): OrchestrationV2Run | null {
  return (
    projection.runs
      .filter((run) => run.status === "completed" && run.checkpointId !== null)
      .toSorted((left, right) => right.ordinal - left.ordinal)[0] ?? null
  );
}

function runForSourcePoint(
  projection: Pick<OrchestrationV2ThreadProjection, "runs" | "checkpoints">,
  sourcePoint: Extract<
    OrchestrationV2Command,
    { readonly type: "thread.fork" | "thread.merge_back" }
  >["sourcePoint"],
): OrchestrationV2Run | null {
  switch (sourcePoint.type) {
    case "latest_stable":
      return latestStableRun(projection);
    case "run":
      return projection.runs.find((run) => run.id === sourcePoint.runId) ?? null;
    case "checkpoint": {
      const checkpoint = projection.checkpoints.find(
        (candidate) => candidate.id === sourcePoint.checkpointId,
      );
      return checkpoint?.runId === null || checkpoint === undefined
        ? null
        : (projection.runs.find((run) => run.id === checkpoint.runId) ?? null);
    }
  }
}

function providerThreadForRun(
  projection: Pick<OrchestrationV2ThreadProjection, "providerThreads">,
  run: OrchestrationV2Run,
): OrchestrationV2ProviderThread | undefined {
  return run.providerThreadId === null
    ? undefined
    : projection.providerThreads.find((candidate) => candidate.id === run.providerThreadId);
}

function providerTurnForRun(
  projection: Pick<OrchestrationV2ThreadProjection, "providerTurns" | "attempts">,
  run: OrchestrationV2Run,
): OrchestrationV2ProviderTurn | undefined {
  if (run.activeAttemptId === null) {
    return undefined;
  }

  return (
    latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId) ??
    projection.providerTurns.find((turn) => {
      const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
      return attempt?.providerTurnId === turn.id;
    })
  );
}

function contextSourcePointForRun(
  projection: Pick<
    OrchestrationV2ThreadProjection,
    "thread" | "providerThreads" | "providerTurns" | "attempts"
  >,
  run: OrchestrationV2Run,
): OrchestrationV2ContextSourcePoint {
  const providerThread = providerThreadForRun(projection, run);
  const providerTurn = providerTurnForRun(projection, run);
  return {
    threadId: projection.thread.id,
    runId: run.id,
    ...(run.checkpointId === null ? {} : { checkpointId: run.checkpointId }),
    ...(providerThread?.nativeThreadRef === null || providerThread?.nativeThreadRef === undefined
      ? {}
      : { providerThreadRef: providerThread.nativeThreadRef }),
    ...(providerTurn?.nativeTurnRef === null || providerTurn?.nativeTurnRef === undefined
      ? {}
      : { providerTurnRef: providerTurn.nativeTurnRef }),
  };
}

function pendingForkTransferForThread(
  projection: Pick<OrchestrationV2ThreadProjection, "contextTransfers" | "thread">,
): OrchestrationV2ContextTransfer | undefined {
  return projection.contextTransfers.find(
    (transfer) =>
      transfer.type === "fork" &&
      transfer.targetThreadId === projection.thread.id &&
      transfer.status === "pending",
  );
}

function pendingMergeBackTransfersForThread(
  projection: Pick<OrchestrationV2ThreadProjection, "contextTransfers" | "thread">,
): ReadonlyArray<OrchestrationV2ContextTransfer> {
  return projection.contextTransfers.filter(
    (transfer) =>
      transfer.type === "merge_back" &&
      transfer.targetThreadId === projection.thread.id &&
      transfer.status === "pending",
  );
}

function latestContextTransfer(
  transfers: ReadonlyArray<OrchestrationV2ContextTransfer>,
): OrchestrationV2ContextTransfer | undefined {
  return transfers.reduce<OrchestrationV2ContextTransfer | undefined>((latest, transfer) => {
    if (latest === undefined) {
      return transfer;
    }
    return DateTime.toEpochMillis(transfer.updatedAt) >= DateTime.toEpochMillis(latest.updatedAt)
      ? transfer
      : latest;
  }, undefined);
}

export function shouldPrepareLegacyImportHandoff(input: {
  readonly hasCompletedRun: boolean;
  readonly historyOrigin: OrchestrationV2AppThread["historyOrigin"];
  readonly legacyImportItemCount: number;
}): boolean {
  return (
    input.historyOrigin === "v1_import" && !input.hasCompletedRun && input.legacyImportItemCount > 0
  );
}

function appendContextHandoffId(
  handoffIds: OrchestrationV2ProviderThread["handoffIds"],
  handoffId: OrchestrationV2ContextHandoff["id"] | null,
): OrchestrationV2ProviderThread["handoffIds"] {
  return handoffId === null ? handoffIds : Array.from(new Set([...handoffIds, handoffId]));
}

/** Ordinal of the latest started fresh-context run. Runs before it never reach a provider again. */
function contextBoundaryOrdinal(
  runs: ReadonlyArray<
    Pick<
      OrchestrationV2Run,
      "ordinal" | "freshContext" | "status" | "startedAt" | "restartContinuationOfRunId"
    >
  >,
): number {
  let boundary = 0;
  for (const run of runs) {
    if (
      run.freshContext === true &&
      run.status !== "queued" &&
      !cancelledBeforeStart(run) &&
      run.ordinal > boundary
    ) {
      boundary = run.ordinal;
    }
  }
  return boundary;
}

/** Threads with no started run (imported, or reserved by another queued run) never cross it. */
function providerThreadPredatesBoundary(
  providerThread: Pick<OrchestrationV2ProviderThread, "firstRunOrdinal">,
  boundary: number,
): boolean {
  return (
    boundary > 0 &&
    (providerThread.firstRunOrdinal === null || providerThread.firstRunOrdinal < boundary)
  );
}

/** A provider that hosts one native thread per session keeps the old conversation alive otherwise. */
function sessionsReleasedByFreshContext(
  projection: Pick<OrchestrationV2ThreadProjection, "providerSessions">,
  activeProviderThread: OrchestrationV2ProviderThread | undefined,
): ReadonlyArray<OrchestrationV2ProviderSession> {
  const sessionId = activeProviderThread?.providerSessionId ?? null;
  return projection.providerSessions.filter(
    (session) =>
      session.id === sessionId &&
      session.status !== "stopped" &&
      session.status !== "error" &&
      !session.capabilities.sessions.supportsMultipleProviderThreadsPerSession,
  );
}

const FRESH_CONTEXT_DETACH_REASON = "Started a fresh session.";

function uniqueSessions(
  sessions: ReadonlyArray<OrchestrationV2ProviderSession>,
): ReadonlyArray<OrchestrationV2ProviderSession> {
  return Array.from(new Map(sessions.map((session) => [session.id, session])).values());
}

function rootProviderThreadsForProvider(
  projection: Pick<OrchestrationV2ThreadProjection, "providerThreads" | "thread" | "runs">,
  providerInstanceId: ModelSelection["instanceId"],
): ReadonlyArray<OrchestrationV2ProviderThread> {
  const boundary = contextBoundaryOrdinal(projection.runs);
  return projection.providerThreads
    .filter(
      (providerThread) =>
        providerThread.providerInstanceId === providerInstanceId &&
        providerThread.appThreadId === projection.thread.id &&
        providerThread.ownerNodeId === null &&
        !providerThreadPredatesBoundary(providerThread, boundary),
    )
    .toSorted(
      (left, right) =>
        (right.lastRunOrdinal ?? 0) - (left.lastRunOrdinal ?? 0) ||
        DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
    );
}

function isHandoffSourceRun(run: OrchestrationV2Run): boolean {
  return run.status === "completed" || run.status === "failed" || run.status === "interrupted";
}

function lastDeliveredRunForProviderThread(
  projection: Pick<OrchestrationV2ThreadProjection, "runs" | "providerTurns">,
  providerThreadId: OrchestrationV2ProviderThread["id"],
): OrchestrationV2Run | undefined {
  return projection.runs.findLast(
    (run) =>
      isHandoffSourceRun(run) &&
      run.providerThreadId === providerThreadId &&
      (run.status === "completed" ||
        projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)),
  );
}

const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(function* () {
  const checkpointService = yield* CheckpointServiceV2;
  const commandPolicy = yield* CommandPolicyV2;
  const contextHandoffService = yield* ContextHandoffServiceV2;
  const eventSink = yield* EventSinkV2;
  const commandReceipts = yield* CommandReceiptStoreV2;
  const idAllocator = yield* IdAllocator.IdAllocatorV2;
  const projects = yield* ProjectStore.ProjectStoreV2;
  const projectionStore = yield* ProjectionStoreV2;
  const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
  const nextTurnItemOrdinal = (
    projection: Pick<OrchestrationV2ThreadProjection, "thread"> &
      Partial<Pick<OrchestrationV2ThreadProjection, "turnItems">>,
  ) =>
    projectionStore.getNextTurnItemOrdinal(projection.thread.id).pipe(
      Effect.mapError(
        (cause) => new OrchestratorProjectionError({ threadId: projection.thread.id, cause }),
      ),
      Effect.map((next) =>
        (projection.turnItems ?? []).reduce(
          (ordinal, item) => Math.max(ordinal, item.ordinal + 1),
          next,
        ),
      ),
    );

  const fileSystem = yield* FileSystem.FileSystem;
  const path = yield* Path.Path;
  const providerAdapters = yield* ProviderAdapterRegistryV2;
  const continuationRequests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
  const providerSessions = yield* ProviderSessionManagerV2;
  const providerSwitchService = yield* ProviderSwitchServiceV2;
  const serverSettings = yield* ServerSettings.ServerSettingsService;
  const runtimePolicy = yield* RuntimePolicyV2;
  const threadForkService = yield* ThreadForkServiceV2;
  const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
  // Read once: the policy is fixed for the server's life, and dispatch takes it as an input.
  const { managedPolicy } = serverSettings;
  const runtimeModeCeiling: RuntimeModeCeiling | undefined =
    managedPolicy?.limits?.maxRuntimeMode === undefined
      ? undefined
      : {
          organization: managedPolicy.organization,
          maxRuntimeMode: managedPolicy.limits.maxRuntimeMode,
        };

  /** The driver when this provider runs `/goal` natively, else undefined. */
  const goalDriverFor = Effect.fn("Orchestrator.goalDriverFor")(function* (
    command: OrchestrationV2ServerCommand,
    providerInstanceId: ProviderInstanceId,
  ) {
    const { driver } = yield* providerAdapters
      .get(providerInstanceId)
      .pipe(mapDispatchError(command));
    return driver === "codex" || driver === "claudeAgent" ? driver : undefined;
  });

  /** Claude Code reads `/goal pause` and `/goal resume` as new goals named "pause" and "resume". */
  const ensureGoalCommandSupported = Effect.fn("Orchestrator.ensureGoalCommandSupported")(
    function* (
      command: OrchestrationV2ServerCommand,
      message: { readonly text: string; readonly attachments: ReadonlyArray<ChatAttachment> },
      providerInstanceId: ProviderInstanceId,
    ) {
      const goalCommand = parseGoalCommand(message);
      if (goalCommand?.type !== "pause" && goalCommand?.type !== "resume") return;
      if ((yield* goalDriverFor(command, providerInstanceId)) !== "claudeAgent") return;
      return yield* new OrchestratorDispatchError({
        commandId: command.commandId,
        commandType: command.type,
        cause: "Claude goals cannot be paused or resumed. Send /goal clear to remove the goal.",
      });
    },
  );

  const mapDispatchError =
    (command: OrchestrationV2ServerCommand) =>
    <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, OrchestratorDispatchError, R> =>
      effect.pipe(
        Effect.mapError(
          (cause) =>
            new OrchestratorDispatchError({
              commandId: command.commandId,
              commandType: command.type,
              cause,
            }),
        ),
      );

  const mapDelegatedCompletionError = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
    effect.pipe(
      Effect.mapError(
        (cause) =>
          new OrchestratorDispatchError({
            commandId: CommandId.make("command:system:delegated-completion-delivery"),
            commandType: "delegated_task.completion-delivery",
            cause,
          }),
      ),
    );

  const providerSessionIdFor = (input: {
    readonly adapter: ProviderAdapter.ProviderAdapterV2["Service"];
    readonly providerInstanceId: ProviderInstanceId;
    readonly threadId: ThreadId;
  }) =>
    input.adapter.getCapabilities().pipe(
      Effect.flatMap((capabilities) =>
        capabilities.sessions.supportsMultipleProviderThreadsPerSession
          ? Effect.succeed(
              idAllocator.derive.providerSession({
                providerInstanceId: input.providerInstanceId,
              }),
            )
          : idAllocator.allocate.providerSession({
              providerInstanceId: input.providerInstanceId,
              threadId: input.threadId,
            }),
      ),
    );

  const makeEvent =
    /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
    Effect.fnUntraced(function* <Event extends OrchestrationV2DomainEvent>(
      command: OrchestrationV2ServerCommand,
      event: Omit<Event, "id">,
    ) {
      const eventId = yield* mapDispatchError(command)(
        idAllocator.allocate.event({
          threadId: event.threadId,
          commandId: command.commandId,
        }),
      );
      return {
        ...event,
        id: eventId,
      } as Event;
    });

  const emit =
    (events: Ref.Ref<Array<OrchestrationV2DomainEvent>>, command: OrchestrationV2ServerCommand) =>
    <Event extends OrchestrationV2DomainEvent>(event: Omit<Event, "id">) =>
      Effect.gen(function* () {
        const withId = yield* makeEvent(command, event);
        yield* Ref.update(events, (existing) => [...existing, withId]);
        return withId;
      });

  // Detaching kills background work running inside the provider process.
  // Probe even with an empty roster: Claude's native subagents never reach
  // pendingBackgroundTasks. Sessions shared across threads probe per thread.
  const sessionHasBackgroundWork = Effect.fn("Orchestrator.sessionHasBackgroundWork")(function* (
    detaching: OrchestrationV2ThreadProjection["providerSessions"][number],
    providerThreads: OrchestrationV2ThreadProjection["providerThreads"],
  ) {
    const session = yield* providerSessions
      .get(detaching.id)
      .pipe(Effect.orElseSucceed(() => Option.none()));
    if (Option.isNone(session)) return false;
    const runtime = session.value;
    const probes =
      !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession &&
      runtime.hasPendingBackgroundWork !== undefined
        ? [runtime.hasPendingBackgroundWork]
        : providerThreads.flatMap((providerThread) =>
            providerThread.providerSessionId === detaching.id &&
            runtime.hasPendingBackgroundWorkForThread !== undefined
              ? [runtime.hasPendingBackgroundWorkForThread(providerThread)]
              : [],
          );
    for (const probe of probes) {
      // A failed probe cannot confirm that the work ended.
      if (yield* probe.pipe(Effect.catchCause(() => Effect.succeed(true)))) return true;
    }
    return false;
  });

  // Runs in every command transaction; avoid reading thousands of finished leaf nodes.
  const readCommandProjection = (threadId: ThreadId) =>
    projectionStore
      .getThreadRecords(
        threadId,
        [
          "runs",
          "attempts",
          "nodes",
          "subagents",
          "providerSessions",
          "providerThreads",
          "providerTurns",
          "runtimeRequests",
          "messages",
          "turnItems",
          "checkpointScopes",
          "contextTransfers",
        ],
        {
          turnItemTypes: ["user_message", "error"],
          messageRoles: ["user"],
          omitSettledLeafNodes: true,
          // Commands here read only undelivered or active tasks; a dispatcher thread keeps
          // thousands of delivered ones, each carrying its full prompt.
          omitSettledSubagents: true,
        },
      )
      .pipe(
        Effect.map((records): OrchestrationV2ThreadProjection => ({
          ...records,
          checkpoints: [],
          plans: [],
          contextHandoffs: [],
          visibleTurnItems: [],
          updatedAt: records.thread.updatedAt,
        })),
        Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
      );

  // Command reads omit settled tasks, but a completion notice counts its whole cohort.
  const notificationTasks = (
    threadId: ThreadId,
    message: Pick<OrchestrationV2ConversationMessage, "delegatedCompletion">,
    loaded: ReadonlyArray<OrchestrationV2Subagent>,
  ) =>
    message.delegatedCompletion === undefined
      ? Effect.succeed(loaded)
      : projectionStore
          .getThreadRecords(threadId, ["subagents"], {
            subagentRunIds: [message.delegatedCompletion.parentRunId],
          })
          .pipe(
            Effect.map((records) => records.subagents),
            Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
          );

  const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
    projectionStore
      .getThreadRecords(
        threadId,
        ["turnItems"],
        runIds === undefined ? undefined : { turnItemRunIds: runIds },
      )
      .pipe(
        Effect.map((records) => records.turnItems),
        Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
      );

  const readInheritedItems = (thread: OrchestrationV2AppThread) =>
    readInheritedHandoffItems(projectionStore, thread).pipe(
      Effect.mapError((cause) => new OrchestratorProjectionError({ threadId: thread.id, cause })),
    );

  /** Includes tombstones, which still reserve their id. */
  const findThread = (threadId: ThreadId) =>
    projectionStore.getThread(threadId).pipe(
      Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
      Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
    );

  /** Side chats live inside their parent, so deleting the parent deletes them too. */
  const liveSideChatIds = Effect.fn("Orchestrator.liveSideChatIds")(
    function* (threadId: ThreadId) {
      const { contextTransfers } = yield* projectionStore
        .getThreadRecords(threadId, ["contextTransfers"])
        .pipe(
          Effect.catchTags({
            ProjectionStoreThreadNotFoundError: () => Effect.succeed({ contextTransfers: [] }),
          }),
        );
      const sideChatIds: Array<ThreadId> = [];
      for (const transfer of contextTransfers) {
        if (transfer.type !== "fork" || transfer.sourceThreadId !== threadId) continue;
        const child = yield* projectionStore
          .getThread(transfer.targetThreadId)
          .pipe(
            Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
          );
        if (child?.lineage.sideChat === true && child.deletedAt === null) {
          sideChatIds.push(child.id);
        }
      }
      return sideChatIds;
    },
    (effect, threadId) =>
      effect.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
  );

  const getProjectionWithPendingEvents =
    /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
    Effect.fnUntraced(function* (
      threadId: ThreadId,
      events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
    ) {
      const pending = (yield* Ref.get(events)).filter((event) => event.threadId === threadId);
      const stored = yield* Effect.option(readCommandProjection(threadId));
      let projection: OrchestrationV2ThreadProjection;
      if (Option.isSome(stored)) {
        projection = stored.value;
      } else {
        const created = pending.find(
          (
            event,
          ): event is Extract<OrchestrationV2DomainEvent, { readonly type: "thread.created" }> =>
            event.type === "thread.created",
        );
        if (created === undefined) {
          return yield* new OrchestratorProjectionError({ threadId });
        }
        projection = emptyProjection(created);
      }

      for (const event of pending) {
        if (event.type === "thread.created" && projection.thread.id === event.payload.id) {
          projection = { ...projection, thread: event.payload, updatedAt: event.occurredAt };
          continue;
        }
        projection = applyToProjection(projection, event);
      }
      return projection;
    });

  const makeSystemEvent =
    /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
    Effect.fnUntraced(function* <Event extends OrchestrationV2DomainEvent>(
      event: Omit<Event, "id">,
    ) {
      const eventId = yield* idAllocator.allocate.event({
        threadId: event.threadId,
      });
      return {
        ...event,
        id: eventId,
      } as Event;
    });

  const writeSystemEvents =
    /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
    Effect.fnUntraced(function* (
      events: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">>,
      effects: ReadonlyArray<PendingOrchestrationEffectV2> = [],
      options: { readonly updateDelegatedCompletionOnly?: boolean } = {},
    ) {
      const withIds = yield* Effect.forEach(events, (event) => makeSystemEvent(event));
      yield* eventSink.writeWithEffects({ events: withIds, effects, ...options });
    });

  const completionDeliveryRun = (
    projection: Pick<OrchestrationV2ThreadProjection, "messages" | "runs">,
    delivery: OrchestrationV2DelegatedCompletionDelivery | null | undefined,
  ) => {
    if (delivery == null) return undefined;
    const message = projection.messages.find((candidate) => candidate.id === delivery.messageId);
    return projection.runs.find((candidate) => candidate.id === message?.runId);
  };

  const completionDeliveryMessage = (
    projection: Pick<OrchestrationV2ThreadProjection, "messages">,
    delivery: OrchestrationV2DelegatedCompletionDelivery | null | undefined,
  ) =>
    delivery === null || delivery === undefined
      ? undefined
      : projection.messages.find((candidate) => candidate.id === delivery.messageId);

  const offerDelegatedCompletionDelivery =
    /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
    Effect.fnUntraced(function* (threadId: ThreadId, parentRunId: RunId) {
      const controls = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
        runIds: [parentRunId],
      });
      const parentRun = controls.runs[0];
      const cohort = parentRun?.delegatedCompletion;
      const delivery = cohort?.delivery;
      if (
        parentRun === undefined ||
        cohort?.disposition !== "open" ||
        delivery == null ||
        delivery.taskIds.length === 0 ||
        controls.thread.archivedAt !== null ||
        controls.thread.deletedAt !== null
      )
        return;
      const messages = yield* projectionStore.getThreadRecords(threadId, ["messages"], {
        messageIds: [delivery.messageId],
      });
      const message = messages.messages[0];
      if (message !== undefined) {
        if (message.delegatedCompletion === undefined || message.runId === null) return;
        const runs = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
          runIds: [message.runId],
        });
        const run = runs.runs[0];
        const turns = yield* projectionStore.getThreadRecords(threadId, ["providerTurns"], {
          providerTurnAttemptIds: run?.activeAttemptId == null ? [] : [run.activeAttemptId],
        });
        if (!isUndeliveredMailboxSteer({ ...messages, ...runs, ...turns }, delivery.messageId))
          return;
      }
      const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"], {
        providerThreadIds: parentRun.providerThreadId === null ? [] : [parentRun.providerThreadId],
      });
      const providerThread =
        parentRun.providerThreadId === null
          ? undefined
          : projection.providerThreads.find(
              (candidate) => candidate.id === parentRun.providerThreadId,
            );
      if (providerThread === undefined) {
        return;
      }
      yield* continuationRequests.offer({
        threadId,
        providerThreadId: providerThread.id,
        driver: providerThread.driver,
        detail: null,
        delivery: "message_text",
        delegatedCompletion: {
          parentRunId,
          generation: delivery.generation,
          messageId: delivery.messageId,
        },
      });
    });

  const offerDelegatedCompletionDeliveries =
    /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
    Effect.fnUntraced(function* (threadId: ThreadId) {
      const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
      for (const run of projection.runs) {
        if (
          run.delegatedCompletion?.delivery !== undefined &&
          run.delegatedCompletion.delivery !== null
        ) {
          yield* offerDelegatedCompletionDelivery(threadId, run.id);
        }
      }
    });

  const emitQueuedRunCancellation = Effect.fn("Orchestrator.emitQueuedRunCancellation")(
    function* (input: {
      readonly command: OrchestrationV2ServerCommand;
      readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
      readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "attempts">;
      readonly run: OrchestrationV2Run;
      readonly now: DateTime.Utc;
    }) {
      const rootNode =
        input.run.rootNodeId === null
          ? undefined
          : input.projection.nodes.find((candidate) => candidate.id === input.run.rootNodeId);
      const attempt =
        input.run.activeAttemptId === null
          ? undefined
          : input.projection.attempts.find(
              (candidate) => candidate.id === input.run.activeAttemptId,
            );
      const emitEvent = emit(input.events, input.command);
      yield* emitEvent({
        type: "run.updated",
        threadId: input.run.threadId,
        runId: input.run.id,
        ...(input.run.rootNodeId === null ? {} : { nodeId: input.run.rootNodeId }),
        providerInstanceId: input.run.providerInstanceId,
        occurredAt: input.now,
        payload: {
          ...input.run,
          status: "cancelled",
          queuePosition: null,
          completedAt: input.now,
        },
      });
      if (attempt !== undefined && rootNode !== undefined) {
        yield* emitEvent({
          type: "run-attempt.updated",
          threadId: input.run.threadId,
          runId: input.run.id,
          nodeId: rootNode.id,
          providerInstanceId: input.run.providerInstanceId,
          occurredAt: input.now,
          payload: {
            ...attempt,
            status: "cancelled",
            completedAt: input.now,
          },
        });
      }
      if (rootNode !== undefined) {
        yield* emitEvent({
          type: "node.updated",
          threadId: input.run.threadId,
          runId: input.run.id,
          nodeId: rootNode.id,
          providerInstanceId: input.run.providerInstanceId,
          occurredAt: input.now,
          payload: {
            ...rootNode,
            status: "cancelled",
            completedAt: input.now,
          },
        });
      }
    },
  );

  const failQueuedRunStart = Effect.fn("Orchestrator.failQueuedRunStart")(function* (
    threadId: ThreadId,
    cause: unknown,
    queuedRunId?: RunId,
  ) {
    const projection = yield* projectionStore.getThreadRecords(
      threadId,
      ["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"],
      { turnItemTypes: [], messageRoles: ["user"] },
    );
    const queuedRun =
      queuedRunId === undefined
        ? nextQueuedRun(projection)
        : projection.runs.find((run) => run.id === queuedRunId && run.status === "queued");
    if (queuedRun === undefined) return;
    const now = yield* DateTime.now;
    const rootNode = projection.nodes.find((node) => node.id === queuedRun.rootNodeId);
    const attempt = projection.attempts.find((entry) => entry.id === queuedRun.activeAttemptId);
    const providerThread = projection.providerThreads.find(
      (entry) => entry.id === queuedRun.providerThreadId,
    );
    const handoffUnsupported =
      typeof cause === "object" &&
      cause !== null &&
      "_tag" in cause &&
      cause._tag === "CommandPolicyCapabilityUnsupportedError";
    const failureCause =
      typeof cause === "object" &&
      cause !== null &&
      "_tag" in cause &&
      cause._tag === "OrchestratorDispatchError" &&
      "cause" in cause
        ? cause.cause
        : cause;
    const failure = makeProviderFailure({
      cause: failureCause,
      code: handoffUnsupported ? "context_handoff_unsupported" : "queued_start_failed",
      class: handoffUnsupported ? "validation_error" : "unknown",
    });
    yield* writeSystemEvents([
      ...(attempt !== undefined && rootNode !== undefined
        ? [
            {
              type: "run-attempt.updated" as const,
              threadId,
              runId: queuedRun.id,
              nodeId: rootNode.id,
              providerInstanceId: queuedRun.providerInstanceId,
              occurredAt: now,
              payload: { ...attempt, status: "failed" as const, completedAt: now },
            },
          ]
        : []),
      ...(rootNode === undefined
        ? []
        : [
            {
              type: "node.updated" as const,
              threadId,
              runId: queuedRun.id,
              nodeId: rootNode.id,
              providerInstanceId: queuedRun.providerInstanceId,
              occurredAt: now,
              payload: { ...rootNode, status: "failed" as const, completedAt: now },
            },
          ]),
      ...(rootNode === undefined || providerThread === undefined
        ? []
        : [
            {
              type: "turn-item.updated" as const,
              threadId,
              runId: queuedRun.id,
              nodeId: rootNode.id,
              providerInstanceId: queuedRun.providerInstanceId,
              occurredAt: now,
              payload: {
                id: idAllocator.derive.turnItemFromProviderItem({
                  driver: providerThread.driver,
                  nativeItemId: `queued-start-failure:${queuedRun.id}`,
                }),
                threadId,
                runId: queuedRun.id,
                nodeId: rootNode.id,
                providerThreadId: providerThread.id,
                providerTurnId: null,
                nativeItemRef: null,
                parentItemId: null,
                ordinal: yield* nextTurnItemOrdinal(projection),
                status: "failed" as const,
                title: "Queued provider could not start",
                startedAt: now,
                completedAt: now,
                updatedAt: now,
                type: "error" as const,
                failure,
              },
            },
          ]),
      {
        type: "run.updated",
        threadId,
        runId: queuedRun.id,
        ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
        providerInstanceId: queuedRun.providerInstanceId,
        occurredAt: now,
        payload: { ...queuedRun, status: "failed", queuePosition: null, completedAt: now },
      },
    ]);
  });

  const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(
    function* (
      threadId: ThreadId,
      options?: {
        readonly failedRunId?: RunId;
        readonly endedRunId?: RunId;
        readonly queuedRunId?: RunId;
      },
    ) {
      if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
      const projection = yield* readCommandProjection(threadId);
      if (
        projection.thread.archivedAt !== null ||
        projection.thread.deletedAt !== null ||
        projection.runs.some(isActiveRun)
      ) {
        return;
      }

      const storedQueuedRun =
        options?.queuedRunId === undefined
          ? nextQueuedRun(projection)
          : projection.runs.find(
              (run) => run.id === options.queuedRunId && run.status === "queued",
            );
      if (storedQueuedRun === undefined) {
        return;
      }
      const queuedMessage = projection.messages.find(
        (candidate) => candidate.id === storedQueuedRun.userMessageId,
      );
      const modelSelection = queuedRunModelSelection(projection, storedQueuedRun);
      const queuedRun = {
        ...storedQueuedRun,
        modelSelection,
        providerInstanceId: modelSelection.instanceId,
      };
      // A limited provider would fail and consume every waiting message.
      if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
        return;
      }
      // Waits without a hold: lowering the thread's mode starts the queue again.
      if (isRuntimeModeAbove(projection.thread.runtimeMode, runtimeModeCeiling?.maxRuntimeMode)) {
        return;
      }
      // Failed worktree creation would run queued work in the project checkout; wait for the user.
      const failedRun = latestExecutedRun(projection.runs);
      const failureClass =
        failedRun?.id === options?.failedRunId
          ? latestRootProviderFailure(failedRun, projection.turnItems)?.class
          : undefined;
      const worktreeMissing =
        failedRun?.id === options?.endedRunId &&
        failedRun?.workspacePreparation?.type === "worktree" &&
        projection.thread.worktreePath === null;
      const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
      const projectOverride =
        settings.projectSettingsOverrides[projection.thread.projectId]?.providerInstances?.[
          queuedRun.providerInstanceId
        ]?.enabled;
      const machineInstance = Object.hasOwn(
        settings.providerInstances,
        queuedRun.providerInstanceId,
      )
        ? settings.providerInstances[queuedRun.providerInstanceId]
        : undefined;
      const instanceDisabledForProject =
        projectOverride !== undefined
          ? !projectOverride