import type {
OrchestrationV2Run,
OrchestrationV2StoredEvent,
RunId,
ThreadId,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Metric from "effect/Metric";
import { normalizeModelMetricLabel } from "../observability/Attributes.ts";
import {
metricAttributes,
runDuration,
runQueueWait,
runsTotal,
} from "../observability/Metrics.ts";
const REMEMBERED_RUNS = 4096;
// A rollback rewrites runs that already finished, stamping the rollback time as their completion,
// so it is not counted as an outcome. Declared here because ThreadManagementService imports
// EventSink.
const OUTCOMES: ReadonlySet<OrchestrationV2Run["status"]> = new Set([
"completed",
"failed",
"cancelled",
"interrupted",
]);
export interface RunOutcome {
readonly runId: RunId;
readonly attributes: ReturnType<typeof metricAttributes>;
readonly queueWait: Duration.Duration | undefined;
readonly duration: Duration.Duration | undefined;
}
/**
* Finished runs can change again; deduplicate outcomes and use lineage for first subagent runs.
* A failed lineage lookup must not block commit or publication.
* `prepare` must read inside the write transaction; `record` must do no I/O under the publish lane,
* or it can deadlock with a writer holding the connection while waiting for that lane.
*/
export const makeRunOutcomeRecorder = <E>(
isSubagentThread: (threadId: ThreadId) => Effect.Effect<boolean, E>,
) => {
const counted = new Set<string>();
// Untraced: both run for every committed batch inside the EventSink operation's span.
// Never reads or writes `counted`, so a transaction that rolls back reserves nothing.
const prepare = Effect.fnUntraced(function* (events: ReadonlyArray<OrchestrationV2StoredEvent>) {
const outcomes: Array<RunOutcome> = [];
// Every finished run is prepared, in order, even one already counted: an earlier outcome in
// this batch may evict it before `record` reaches it, so only `record` decides what counts.
// Only the lineage read is shared.
const delegatedThreads = new Map<ThreadId, boolean>();
for (const { event } of events) {
if (event.type !== "run.updated") continue;
const run = event.payload;
if (!OUTCOMES.has(run.status)) continue;
let delegated = run.delegatedTaskId !== undefined || delegatedThreads.get(run.threadId);
if (delegated === undefined) {
delegated = yield* isSubagentThread(run.threadId).pipe(Effect.orElseSucceed(() => false));
delegatedThreads.set(run.threadId, delegated);
}
outcomes.push({
runId: run.id,
attributes: metricAttributes({
provider: run.providerInstanceId,
modelFamily: normalizeModelMetricLabel(run.modelSelection.model),
outcome: run.status,
delegated,
}),
queueWait:
run.startedAt === null ? undefined : DateTime.distance(run.requestedAt, run.startedAt),
duration:
run.startedAt === null || run.completedAt === null
? undefined
: DateTime.distance(run.startedAt, run.completedAt),
});
}
return outcomes;
});
// Uninterruptible so a run is never marked counted with only part of its measurements.
const record = Effect.fnUntraced(function* (outcomes: ReadonlyArray<RunOutcome>) {
for (const outcome of outcomes) {
// Concurrent writers can prepare the same run; the first to publish counts it.
if (counted.has(outcome.runId)) continue;
counted.add(outcome.runId);
if (counted.size > REMEMBERED_RUNS) counted.delete(counted.values().next().value!);
yield* Metric.update(Metric.withAttributes(runsTotal, outcome.attributes), 1);
if (outcome.queueWait !== undefined) {
yield* Metric.update(
Metric.withAttributes(runQueueWait, outcome.attributes),
outcome.queueWait,
);
}
if (outcome.duration !== undefined) {
yield* Metric.update(
Metric.withAttributes(runDuration, outcome.attributes),
outcome.duration,
);
}
}
}, Effect.uninterruptible);
return { prepare, record };
};