sailfork / t3code-github / apps/ server/ src/ telemetry/ ProviderUsageAnalytics.test.ts

import {
  NodeId,
  ProviderInstanceId,
  ProviderThreadId,
  ProviderTurnId,
  RunId,
  type OrchestrationV2ProviderTurn,
  type ServerProvider,
  type TurnTokenUsage,
} from "@t3tools/contracts";
import { describe, expect, it } from "@effect/vitest";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";

import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as AnalyticsService from "./AnalyticsService.ts";
import { catalogModelLabel } from "./ProviderDimensions.ts";
import {
  ProviderTurnAnalytics,
  RequestUsageLedger,
  layer,
  providerTurnProperties,
  type ProviderTurnAnalyticsContext,
  type ProviderTurnSegment,
} from "./ProviderUsageAnalytics.ts";

const providers = [
  // Codex lists whatever its configured model provider serves, including private deployments.
  { driver: "codex", models: [{ slug: "gpt-6-sol" }, { slug: "acme-azure-deployment" }] },
  { driver: "claudeAgent", models: [{ slug: "claude-opus-5-5" }, { slug: "claude-haiku-4-5" }] },
  { driver: "opencode", models: [{ slug: "ollama/acme-internal" }] },
].map(
  ({ driver, models }) =>
    ({
      instanceId: ProviderInstanceId.make(driver),
      driver,
      models: models.map((model) => ({ ...model, isCustom: false })),
    }) as unknown as ServerProvider,
);
const label = (model: string) => catalogModelLabel(providers, model);

const at = (ms: number) => DateTime.makeUnsafe(ms);
const complete = (input: number, output: number, extra: Partial<TurnTokenUsage> = {}) =>
  ({
    usageStatus: "complete",
    usageScope: "main_agent",
    hasSubagents: false,
    inputTokens: input,
    outputTokens: output,
    ...extra,
  }) as TurnTokenUsage;

const turn = (
  id: string,
  usage: TurnTokenUsage | undefined,
  status: OrchestrationV2ProviderTurn["status"] = "completed",
): OrchestrationV2ProviderTurn => ({
  id: ProviderTurnId.make(id),
  providerThreadId: ProviderThreadId.make("provider-thread"),
  nodeId: NodeId.make(`node:${id}`),
  runAttemptId: null,
  nativeTurnRef: null,
  ordinal: 1,
  status,
  startedAt: at(1_000),
  completedAt: at(4_000),
  ...(usage === undefined ? {} : { turnTokenUsage: usage }),
});

const context = (
  runId: string,
  segment: ProviderTurnSegment,
  model = "gpt-6-sol",
): ProviderTurnAnalyticsContext => ({
  runId: RunId.make(runId),
  modelSelection: {
    instanceId: ProviderInstanceId.make("codex"),
    model,
    options: [
      { id: "reasoningEffort", value: "xhigh" },
      { id: "serviceTier", value: "priority" },
    ],
  },
  runtimeMode: "full-access",
  interactionMode: "default",
  origin: { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
  segment,
});

const settled = (runId: string, childrenRunning = false) => ({
  runId: RunId.make(runId),
  status: "completed" as const,
  completedAt: at(9_000),
  childrenRunning,
});

describe("provider usage reporting", () => {
  it.effect(
    "records one request across a continuation, a child, a replay, and a late child",
    () => {
      const recorded: Array<readonly [string, unknown]> = [];
      const layerTest = layer.pipe(
        Layer.provide(
          Layer.mergeAll(
            Layer.mock(AnalyticsService.AnalyticsService)({
              record: (event, properties) =>
                Effect.sync(() => void recorded.push([event, properties])),
            }),
            Layer.mock(ProviderRegistry.ProviderRegistry)({
              getProviders: Effect.succeed(providers),
            }),
          ),
        ),
      );
      return Effect.gen(function* () {
        const analytics = yield* ProviderTurnAnalytics;
        const root = context("run-a", "initial");
        yield* analytics.runStarted({ runId: RunId.make("run-a"), driver: "codex", context: root });
        const first = { driver: "codex", context: root };
        const firstTurn = turn("root", complete(100, 10, { models: ["gpt-6-sol"] }));
        yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
        yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
        yield* analytics.turnCompleted({
          driver: "codex",
          context: context("run-a", "continuation"),
          providerTurn: turn("goal", complete(50, 5, { models: ["gpt-6-sol"] })),
        });
        yield* analytics.turnCompleted({
          driver: "codex",
          context: context("run-a", "native_child"),
          providerTurn: turn("child", complete(20, 2)),
        });
        yield* analytics.runSettled(settled("run-a"));
        yield* analytics.turnCompleted({
          driver: "codex",
          context: context("run-a", "native_child"),
          providerTurn: turn("late-child", complete(7, 1)),
        });
        yield* analytics.runSettled(settled("run-a"));
        yield* analytics.runStarted({
          runId: RunId.make("run-b"),
          driver: "codex",
          context: context("run-b", "initial"),
        });
        yield* analytics.runSettled(settled("run-b"));

        const requests = recorded.filter(([event]) => event === "provider.request.completed");
        expect(recorded.filter(([event]) => event === "provider.turn.completed")).toHaveLength(5);
        expect(requests).toHaveLength(2);
        // The whole payload: no run, turn, or thread ids, prompts, or raw model names.
        expect(requests[0]?.[1]).toEqual({
          reportVersion: 2,
          provider: "codex",
          requestKind: "user",
          terminalStatus: "completed",
          lifecycleObserved: true,
          model: "gpt-6-sol",
          effort: "xhigh",
          serviceTier: "fast",
          runtimeMode: "full-access",
          interactionMode: "default",
          createdBy: "user",
          creationSource: "web",
          scheduled: false,
          subagent: false,
          providerTurns: 2,
          continuations: 1,
          steeringRestarts: 0,
          usageStatus: "complete",
          inputTokens: 150,
          outputTokens: 15,
          nativeChildTurns: 1,
          childUsageStatus: "complete",
          childInputTokens: 20,
          childOutputTokens: 2,
          durationMs: 8_000,
        });
        expect(requests[1]?.[1]).toMatchObject({ providerTurns: 0, usageStatus: "unavailable" });
      }).pipe(Effect.provide(layerTest));
    },
  );

  it("keeps a request partial when its start, a steered segment, or a counter is missing", () => {
    const ledger = new RequestUsageLedger();
    ledger.start(RunId.make("restarted"), "codex", {
      ...context("restarted", "initial"),
      restartContinuation: true,
    });
    ledger.add({
      driver: "codex",
      context: context("restarted", "initial"),
      providerTurn: turn("after-restart", complete(10, 1)),
    });
    expect(ledger.settle(settled("restarted"), label)).toMatchObject({
      restartContinuation: true,
      lifecycleObserved: false,
      usageStatus: "partial",
      inputTokens: 10,
    });

    ledger.start(RunId.make("steered"), "claudeAgent", context("steered", "initial"));
    ledger.add({
      driver: "claudeAgent",
      context: context("steered", "initial"),
      providerTurn: turn(
        "steered-away",
        { usageStatus: "partial", usageScope: "main_agent", hasSubagents: false, outputTokens: 4 },
        "interrupted",
      ),
    });
    ledger.add({
      driver: "claudeAgent",
      context: context("steered", "steering_restart"),
      providerTurn: turn("steered-to", complete(30, 6)),
    });
    expect(ledger.settle(settled("steered"), label)).toMatchObject({
      usageStatus: "partial",
      steeringRestarts: 1,
      inputTokens: 30,
      outputTokens: 10,
    });

    ledger.start(RunId.make("silent"), "grok", context("silent", "initial"));
    ledger.add({
      driver: "grok",
      context: context("silent", "initial"),
      providerTurn: turn("silent", undefined),
    });
    const silent = ledger.settle(settled("silent", true), label);
    expect(silent).toMatchObject({
      usageStatus: "unavailable",
      childrenRunning: true,
      childUsageStatus: "unavailable",
    });
    expect(silent).not.toHaveProperty("outputTokens");
  });

  it("confirms only the models a provider reported", () => {
    const opus = context("models", "initial", "claude-opus-5-5");
    const properties = (usage: TurnTokenUsage | undefined) =>
      providerTurnProperties(
        { driver: "claudeAgent", context: opus, providerTurn: turn("t", usage) },
        label,
      );
    expect(properties(complete(1, 1, { models: ["claude-haiku-4-5-20251001"] }))).toMatchObject({
      model: "claude-opus-5-5",
      observedModel: "claude-haiku-4-5",
      mixedModels: false,
    });
    const unconfirmed = properties(complete(1, 1));
    expect(unconfirmed).toMatchObject({ model: "claude-opus-5-5" });
    expect(unconfirmed).not.toHaveProperty("mixedModels");
    expect(properties(complete(1, 1, { models: ["a", "b"] }))).toMatchObject({ mixedModels: true });

    const ledger = new RequestUsageLedger();
    ledger.start(RunId.make("mixed"), "codex", context("mixed", "initial"));
    ledger.add({
      driver: "codex",
      context: context("mixed", "initial"),
      providerTurn: turn("root", complete(1, 1, { models: ["gpt-6-sol"] })),
    });
    ledger.add({
      driver: "codex",
      context: context("mixed", "native_child"),
      providerTurn: turn("child", complete(1, 1, { models: ["gpt-6-luna"] })),
    });
    expect(ledger.settle(settled("mixed"), label)).toMatchObject({ mixedModels: true });
  });

  it("keeps zero, absent, local-command, and ledger usage distinct", () => {
    const properties = (usage: TurnTokenUsage | undefined) =>
      providerTurnProperties(
        { driver: "claudeAgent", context: context("z", "initial"), providerTurn: turn("t", usage) },
        label,
      );
    expect(properties(complete(0, 0))).toMatchObject({
      usageStatus: "complete",
      usageReported: true,
      inputTokens: 0,
      outputTokens: 0,
    });
    const absent = properties(undefined);
    expect(absent).toMatchObject({ usageStatus: "unavailable", usageReported: false });
    expect(absent).not.toHaveProperty("outputTokens");
    expect(
      properties(
        complete(0, 0, { localCommand: true, otherUsage: { inputTokens: 900, outputTokens: 80 } }),
      ),
    ).toMatchObject({ localCommand: true, otherInputTokens: 900, otherOutputTokens: 80 });
    expect(properties(complete(74_167, 236, { reasoningTokens: 30 }))).toMatchObject({
      outputTokens: 236,
      reasoningTokens: 30,
    });
  });

  it("reports routed catalog models and closed efforts only", () => {
    const routed = (model: string, variant: string) =>
      providerTurnProperties(
        {
          driver: "opencode",
          context: {
            modelSelection: {
              instanceId: ProviderInstanceId.make("opencode"),
              model,
              options: [{ id: "variant", value: variant }],
            },
          },
          providerTurn: turn("t", complete(1, 1, { models: [model] })),
        },
        label,
      );
    expect(routed("my-proxy/claude-opus-5-5", "high")).toMatchObject({
      model: "claude-opus-5-5",
      observedModel: "claude-opus-5-5",
      effort: "high",
    });
    for (const local of [
      routed("ollama/acme-internal", "secret-variant"),
      routed("acme-azure-deployment", "high"),
    ]) {
      expect(local).not.toHaveProperty("model");
      expect(local).not.toHaveProperty("observedModel");
    }
    expect(routed("ollama/acme-internal", "secret-variant")).not.toHaveProperty("effort");
  });

  it("reports closed service tiers from a tier option or a fast mode toggle", () => {
    const tier = (option: { id: string; value: string | boolean }) =>
      providerTurnProperties(
        {
          driver: "codex",
          context: {
            modelSelection: {
              instanceId: ProviderInstanceId.make("codex"),
              model: "gpt-6-sol",
              options: [option],
            },
          },
          providerTurn: turn("t", complete(1, 1)),
        },
        label,
      ).serviceTier;
    expect(tier({ id: "serviceTier", value: "ultrafast" })).toBe("ultrafast");
    expect(tier({ id: "serviceTier", value: "default" })).toBe("standard");
    expect(tier({ id: "serviceTier", value: "/home/me/secret" })).toBeUndefined();
    expect(tier({ id: "fastMode", value: true })).toBe("fast");
    expect(tier({ id: "fastMode", value: false })).toBe("standard");
  });
});