sailfork / t3code-github / apps/ server/ src/ telemetry/ ProviderUsageAnalytics.ts

/**
 * Provider token reporting: one `provider.turn.completed` per provider turn and one
 * `provider.request.completed` per run, built from the same turns so each counts once.
 *
 * @module ProviderUsageAnalytics
 */
import type {
  ModelSelection,
  OrchestrationV2Actor,
  OrchestrationV2CreationSource,
  OrchestrationV2ProviderTurn,
  OrchestrationV2RunAttempt,
  ProviderInteractionMode,
  RunId,
  RuntimeMode,
  TurnTokenUsage,
} from "@t3tools/contracts";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";

import {
  getModelSelectionBooleanOptionValue,
  getModelSelectionStringOptionValue,
} from "@t3tools/shared/model";
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as AnalyticsService from "./AnalyticsService.ts";
import { catalogModelLabel } from "./ProviderDimensions.ts";

// Bump when a field changes meaning, so reports can split old and new rows.
const REPORT_VERSION = 2;

/**
 * Where the run's starting message came from. Most turns start without a
 * person (provider continuations, agent sends, delegated subagents), so
 * reports need this to tell user turns apart.
 */
export interface ProviderTurnOrigin {
  readonly createdBy: OrchestrationV2Actor;
  readonly creationSource: OrchestrationV2CreationSource;
  readonly scheduled: boolean;
  readonly subagent: boolean;
}

/** What started this provider turn inside its run. */
export type ProviderTurnSegment =
  | OrchestrationV2RunAttempt["reason"]
  | "continuation"
  | "native_child";

export interface ProviderTurnAnalyticsContext {
  readonly runId?: RunId;
  readonly modelSelection: ModelSelection;
  readonly runtimeMode?: RuntimeMode;
  readonly interactionMode?: ProviderInteractionMode;
  readonly origin?: ProviderTurnOrigin;
  readonly segment?: ProviderTurnSegment;
  readonly restartContinuation?: boolean;
}

export interface ProviderTurnReport {
  readonly driver: string;
  readonly providerTurn: OrchestrationV2ProviderTurn;
  readonly context?: ProviderTurnAnalyticsContext;
}

export interface RunSettledReport {
  readonly runId: RunId;
  readonly status: OrchestrationV2ProviderTurn["status"];
  readonly completedAt: DateTime.Utc;
  // Run-owned children were still open when the run settled; their later turns are not added.
  readonly childrenRunning: boolean;
}

type Properties = Readonly<Record<string, unknown>>;
/** Maps a native model id to a vendor catalog label, or undefined to keep it out. */
export type ModelLabeler = (model: string) => string | undefined;

const TERMINAL_STATUSES: ReadonlySet<OrchestrationV2ProviderTurn["status"]> = new Set([
  "completed",
  "failed",
  "interrupted",
  "cancelled",
]);

const REPORTED_EFFORTS: ReadonlySet<string> = new Set([
  "none",
  "minimal",
  "low",
  "medium",
  "high",
  "xhigh",
  "max",
  "ultrathink",
  "ultracode",
  "brutal",
]);

const defined = (entries: Properties): Properties =>
  Object.fromEntries(Object.entries(entries).filter(([, value]) => value !== undefined));

// OpenCode names its effort option `variant`, and variants can be user-defined.
function selectedEffort(selection: ModelSelection): string | undefined {
  const effort =
    getModelSelectionStringOptionValue(selection, "reasoningEffort") ??
    getModelSelectionStringOptionValue(selection, "effort") ??
    getModelSelectionStringOptionValue(selection, "variant");
  return effort !== undefined && REPORTED_EFFORTS.has(effort) ? effort : undefined;
}

// Codex sends a tier id (its "fast" config shows up as "priority"); Claude has a fastMode toggle.
const REPORTED_SERVICE_TIERS: ReadonlyMap<string, string> = new Map([
  ["default", "standard"],
  ["flex", "flex"],
  ["fast", "fast"],
  ["priority", "fast"],
  ["ultrafast", "ultrafast"],
]);

function selectedServiceTier(selection: ModelSelection): string | undefined {
  const tier = getModelSelectionStringOptionValue(selection, "serviceTier");
  if (tier !== undefined) return REPORTED_SERVICE_TIERS.get(tier);
  const fastMode = getModelSelectionBooleanOptionValue(selection, "fastMode");
  return fastMode === undefined ? undefined : fastMode ? "fast" : "standard";
}

/**
 * Confirmed models across counted work. Unknown when any piece of work has no
 * reported model, unless two distinct models were already seen.
 */
function modelObservation(
  models: ReadonlyArray<ReadonlyArray<string> | undefined>,
  label: ModelLabeler,
): Properties {
  const distinct = new Set(models.flatMap((entry) => entry ?? []));
  if (distinct.size > 1) return { mixedModels: true };
  if (distinct.size === 0 || models.some((entry) => entry === undefined || entry.length === 0)) {
    return {};
  }
  return defined({ mixedModels: false, observedModel: label([...distinct][0]!) });
}

function selectionProperties(
  context: ProviderTurnAnalyticsContext | undefined,
  label: ModelLabeler,
) {
  const selection = context?.modelSelection;
  return defined({
    model: selection === undefined ? undefined : label(selection.model),
    effort: selection === undefined ? undefined : selectedEffort(selection),
    serviceTier: selection === undefined ? undefined : selectedServiceTier(selection),
    runtimeMode: context?.runtimeMode,
    interactionMode: context?.interactionMode,
    ...context?.origin,
  });
}

function childKind(context: ProviderTurnAnalyticsContext | undefined) {
  if (context?.segment === "native_child") return "native";
  return context?.origin?.subagent === true ? "t3" : undefined;
}

function counters(usage: TurnTokenUsage | undefined): Properties {
  return defined({
    inputTokens: usage?.inputTokens,
    cachedInputTokens: usage?.cachedInputTokens,
    cacheCreationTokens: usage?.cacheCreationTokens,
    outputTokens: usage?.outputTokens,
    reasoningTokens: usage?.reasoningTokens,
  });
}

export function providerTurnProperties(
  report: ProviderTurnReport,
  label: ModelLabeler,
): Properties {
  const { providerTurn, context } = report;
  const usage = providerTurn.turnTokenUsage;
  return defined({
    reportVersion: REPORT_VERSION,
    provider: report.driver,
    terminalStatus: providerTurn.status,
    usageStatus: usage?.usageStatus ?? "unavailable",
    usageScope: usage?.usageScope ?? "main_agent",
    usageReported: usage !== undefined,
    hasSubagents: usage?.hasSubagents,
    ...counters(usage),
    localCommand: usage?.localCommand === true ? true : undefined,
    otherInputTokens: usage?.otherUsage?.inputTokens,
    otherOutputTokens: usage?.otherUsage?.outputTokens,
    ...selectionProperties(context, label),
    ...modelObservation([usage?.models], label),
    segment: context?.segment,
    childKind: childKind(context),
    durationMs:
      providerTurn.startedAt && providerTurn.completedAt
        ? Math.max(
            0,
            DateTime.toEpochMillis(providerTurn.completedAt) -
              DateTime.toEpochMillis(providerTurn.startedAt),
          )
        : undefined,
  });
}

/** How the request reached the provider; only `user` counts as a person asking. */
function requestKind(origin: ProviderTurnOrigin | undefined) {
  if (origin === undefined) return "unknown";
  if (origin.scheduled) return "scheduled";
  if (origin.subagent) return "t3_child";
  if (origin.createdBy === "user") return "user";
  if (origin.creationSource === "provider") return "provider_continuation";
  return origin.createdBy;
}

function sumOf(
  usages: ReadonlyArray<TurnTokenUsage>,
  field:
    | "inputTokens"
    | "cachedInputTokens"
    | "cacheCreationTokens"
    | "outputTokens"
    | "reasoningTokens",
  requireAll: boolean,
): number | undefined {
  const values = usages.map((usage) => usage[field]);
  const known = values.filter((value) => value !== undefined);
  if (known.length === 0 || (requireAll && known.length < values.length)) return undefined;
  return known.reduce((sum, value) => sum + value, 0);
}

/** complete only when every turn is complete; partial when any turn has counts. */
function combinedStatus(usages: ReadonlyArray<TurnTokenUsage | undefined>) {
  if (usages.length > 0 && usages.every((usage) => usage?.usageStatus === "complete")) {
    return "complete";
  }
  return usages.some(
    (usage) => usage?.inputTokens !== undefined || usage?.outputTokens !== undefined,
  )
    ? "partial"
    : "unavailable";
}

interface RequestTurn {
  readonly child: boolean;
  readonly segment: ProviderTurnSegment | undefined;
  readonly providerTurn: OrchestrationV2ProviderTurn;
}

interface RequestEntry {
  driver: string | undefined;
  context: ProviderTurnAnalyticsContext | undefined;
  readonly lifecycleObserved: boolean;
  readonly turns: Map<string, RequestTurn>;
}

const MAX_TRACKED_RUNS = 2048;

/**
 * Collects terminal provider turns per run in this process. A replayed turn
 * replaces its earlier copy, a settled run ignores late turns, and a run whose
 * initial attempt started before this process reports `lifecycleObserved: false`.
 */
export class RequestUsageLedger {
  private readonly open = new Map<RunId, RequestEntry>();
  private readonly settled = new Set<RunId>();

  start(runId: RunId, driver: string, context: ProviderTurnAnalyticsContext): void {
    if (this.settled.has(runId) || this.open.has(runId)) return;
    // A restart continuation carries on a run whose earlier turns this process never saw.
    const lifecycleObserved = context.restartContinuation !== true;
    this.open.set(runId, { driver, context, lifecycleObserved, turns: new Map() });
    evictOldest(this.open);
  }

  add(report: ProviderTurnReport): void {
    const runId = report.context?.runId;
    if (runId === undefined || this.settled.has(runId)) return;
    if (!TERMINAL_STATUSES.has(report.providerTurn.status)) return;
    let entry = this.open.get(runId);
    if (entry === undefined) {
      entry = { driver: undefined, context: undefined, lifecycleObserved: false, turns: new Map() };
      this.open.set(runId, entry);
      evictOldest(this.open);
    }
    const child = report.context?.segment === "native_child";
    if (!child) {
      entry.driver ??= report.driver;
      entry.context ??= report.context;
    }
    entry.turns.set(String(report.providerTurn.id), {
      child,
      segment: report.context?.segment,
      providerTurn: report.providerTurn,
    });
  }

  settle(report: RunSettledReport, label: ModelLabeler): Properties | undefined {
    if (this.settled.has(report.runId)) return undefined;
    this.settled.add(report.runId);
    evictOldest(this.settled);
    const entry = this.open.get(report.runId);
    this.open.delete(report.runId);
    if (entry === undefined) return undefined;

    const turns = [...entry.turns.values()];
    const root = turns.filter((turn) => !turn.child);
    const children = turns.filter((turn) => turn.child);
    const rootUsage = root.map((turn) => turn.providerTurn.turnTokenUsage);
    const childUsage = children.map((turn) => turn.providerTurn.turnTokenUsage);
    const { lifecycleObserved } = entry;
    const measuredRoot = rootUsage.filter((usage) => usage !== undefined);
    const status = combinedStatus(rootUsage);
    const rootComplete = status === "complete";
    const otherUsage = measuredRoot.map((usage) => usage.otherUsage);
    const allOtherKnown = otherUsage.length > 0 && otherUsage.every((usage) => usage !== undefined);
    const anyRootSubagents = measuredRoot.some((usage) => usage.hasSubagents);
    const measuredChildren = childUsage.filter((usage) => usage !== undefined);
    const childStatus =
      children.length === 0 && !anyRootSubagents && !report.childrenRunning
        ? "none"
        : children.length === 0
          ? "unavailable"
          : report.childrenRunning && combinedStatus(childUsage) === "complete"
            ? "partial"
            : combinedStatus(childUsage);
    const startedAt = turns
      .map((turn) => turn.providerTurn.startedAt)
      .filter((value) => value !== null)
      .map(DateTime.toEpochMillis);

    return defined({
      reportVersion: REPORT_VERSION,
      provider: entry.driver ?? "unknown",
      requestKind: requestKind(entry.context?.origin),
      restartContinuation: entry.context?.restartContinuation === true ? true : undefined,
      terminalStatus: report.status,
      lifecycleObserved,
      ...selectionProperties(entry.context, label),
      providerTurns: root.length,
      continuations: root.filter((turn) => turn.segment === "continuation").length,
      steeringRestarts: root.filter((turn) => turn.segment === "steering_restart").length,
      usageStatus: lifecycleObserved || status === "unavailable" ? status : "partial",
      inputTokens: sumOf(measuredRoot, "inputTokens", rootComplete),
      cachedInputTokens: sumOf(measuredRoot, "cachedInputTokens", true),
      cacheCreationTokens: sumOf(measuredRoot, "cacheCreationTokens", true),
      outputTokens: sumOf(measuredRoot, "outputTokens", rootComplete),
      reasoningTokens: sumOf(measuredRoot, "reasoningTokens", true),
      localCommand: measuredRoot.some((usage) => usage.localCommand === true) ? true : undefined,
      otherInputTokens: allOtherKnown
        ? otherUsage.reduce((sum, usage) => sum + (usage?.inputTokens ?? 0), 0)
        : undefined,
      otherOutputTokens: allOtherKnown
        ? otherUsage.reduce((sum, usage) => sum + (usage?.outputTokens ?? 0), 0)
        : undefined,
      nativeChildTurns: children.length,
      childUsageStatus: childStatus,
      childInputTokens: sumOf(measuredChildren, "inputTokens", false),
      childOutputTokens: sumOf(measuredChildren, "outputTokens", false),
      childrenRunning: report.childrenRunning ? true : undefined,
      ...modelObservation(
        [...rootUsage, ...childUsage].map((usage) => usage?.models),
        label,
      ),
      durationMs:
        startedAt.length === 0
          ? undefined
          : Math.max(0, DateTime.toEpochMillis(report.completedAt) - Math.min(...startedAt)),
    });
  }
}

function evictOldest(collection: Map<RunId, unknown> | Set<RunId>) {
  if (collection.size <= MAX_TRACKED_RUNS) return;
  const oldest = collection.keys().next().value;
  if (oldest !== undefined) collection.delete(oldest);
}

export class ProviderTurnAnalytics extends Context.Reference<{
  readonly runStarted: (input: {
    readonly runId: RunId;
    readonly driver: string;
    readonly context: ProviderTurnAnalyticsContext;
  }) => Effect.Effect<void>;
  readonly turnCompleted: (report: ProviderTurnReport) => Effect.Effect<void>;
  readonly runSettled: (report: RunSettledReport) => Effect.Effect<void>;
}>("t3/telemetry/ProviderTurnAnalytics", {
  defaultValue: () => ({
    runStarted: () => Effect.void,
    turnCompleted: () => Effect.void,
    runSettled: () => Effect.void,
  }),
}) {}

export const layer = Layer.effect(
  ProviderTurnAnalytics,
  Effect.gen(function* () {
    const analytics = yield* AnalyticsService.AnalyticsService;
    const registry = yield* ProviderRegistry.ProviderRegistry;
    const ledger = new RequestUsageLedger();
    const labeler = Effect.map(
      registry.getProviders,
      (providers): ModelLabeler =>
        (model) =>
          catalogModelLabel(providers, model),
    );
    return {
      runStarted: Effect.fn("ProviderTurnAnalytics.runStarted")((input) =>
        Effect.sync(() => ledger.start(input.runId, input.driver, input.context)),
      ),
      turnCompleted: Effect.fn("ProviderTurnAnalytics.turnCompleted")(function* (report) {
        ledger.add(report);
        yield* analytics.record(
          "provider.turn.completed",
          providerTurnProperties(report, yield* labeler),
        );
      }),
      runSettled: Effect.fn("ProviderTurnAnalytics.runSettled")(function* (report) {
        const properties = ledger.settle(report, yield* labeler);
        if (properties !== undefined) {
          yield* analytics.record("provider.request.completed", properties);
        }
      }),
    };
  }),
);