sailfork / t3code-github / apps/ server/ src/ usage/ usageTranscriptReader.ts

// @effect-diagnostics nodeBuiltinImport:off
// Cold windows span ~1.4 GB/1,500 files; node:fs buffer streaming is ~10x cheaper than materializing files.
import * as NodeFSP from "node:fs/promises";
import * as NodePath from "node:path";
import * as NodeStringDecoder from "node:string_decoder";
import * as NodeZlib from "node:zlib";

import type {
  SelectedFields,
  TranscriptUsageFormat,
  UsageRecord,
} from "@t3tools/provider-core/server/usage";
import * as Option from "effect/Option";
import * as Schema from "effect/Schema";

import { createTranscriptJsonReader } from "../project/AgentSessionJson.ts";

export interface TranscriptFile {
  readonly path: string;
  readonly size: number;
  readonly mtimeMs: number;
}

/** Resume only if bytes before resumeOffset match the guard hash; rewritten files would corrupt totals. */
export interface TranscriptParsePosition {
  readonly resumeOffset: number;
  readonly guardLength: number;
  readonly guardHash: number;
  readonly state: unknown;
}

export interface TranscriptParseResult {
  readonly records: readonly UsageRecord[];
  /** Unterminated tail records stay outside records/position so the next scan rereads them. */
  readonly tailRecords: readonly UsageRecord[];
  /** Includes tentative tail diagnostics; position.state only includes committed lines. */
  readonly malformedRecords: number;
  readonly position: TranscriptParsePosition;
  readonly resumed: boolean;
}

export const GUARD_LENGTH = 64;
const STREAMING_THRESHOLD_BYTES = 8 * 1024 * 1024;
const NEWLINE = 0x0a;
const STAT_CONCURRENCY = 32;
const CARRIAGE_RETURN = 0x0d;

function selectUsageFields(fields: SelectedFields) {
  return (path: ReadonlyArray<string | number | null>): boolean => {
    let selected: true | SelectedFields = fields;
    for (const key of path) {
      if (selected === true) return true;
      if (typeof key !== "string" || !Object.hasOwn(selected, key)) return false;
      selected = selected[key]!;
    }
    return true;
  };
}

function fnv1a(buffer: Buffer): number {
  let hash = 0x811c9dc5;
  for (let index = 0; index < buffer.length; index += 1) {
    hash ^= buffer[index]!;
    hash = Math.imul(hash, 0x01000193);
  }
  return hash >>> 0;
}

/** Skips Grok logs without usage; preserves readdir order for first-seen deduplication. */
export async function listTranscriptFiles(
  root: string,
  sinceMs: number,
  options?: { readonly fileName?: string },
): Promise<{ readonly files: readonly TranscriptFile[]; readonly failedPaths: number }> {
  const fileName = options?.fileName;
  const candidates: string[] = [];
  let failedPaths = 0;
  const walk = async (dir: string): Promise<void> => {
    let entries;
    try {
      entries = await NodeFSP.readdir(dir, { withFileTypes: true });
    } catch {
      failedPaths += 1;
      return;
    }
    for (const entry of entries) {
      const child = NodePath.join(dir, entry.name);
      if (entry.isDirectory()) await walk(child);
      else if (
        fileName !== undefined
          ? entry.name === fileName
          : entry.name.endsWith(".jsonl") || entry.name.endsWith(".jsonl.zst")
      ) {
        candidates.push(child);
      }
    }
  };
  await walk(root);

  const found: Array<TranscriptFile | undefined> = Array.from({ length: candidates.length });
  const queue = candidates.entries();
  const statQueued = async (): Promise<void> => {
    for (const [index, path] of queue) {
      try {
        const stats = await NodeFSP.stat(path);
        if (stats.mtimeMs >= sinceMs) {
          found[index] = { path, size: stats.size, mtimeMs: stats.mtimeMs };
        }
      } catch (error) {
        if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
          failedPaths += 1;
        }
      }
    }
  };
  await Promise.all(
    Array.from({ length: Math.min(STAT_CONCURRENCY, candidates.length) }, statQueued),
  );
  return { files: found.filter((file) => file !== undefined), failedPaths };
}

export async function readDirectoryVolumeId(path: string): Promise<string> {
  try {
    const stats = await NodeFSP.stat(path);
    return `${stats.dev}:${stats.ino}`;
  } catch {
    return "";
  }
}

async function guardMatches(
  handle: NodeFSP.FileHandle,
  position: TranscriptParsePosition,
): Promise<boolean> {
  if (position.guardLength <= 0 || position.guardLength > GUARD_LENGTH) return false;
  try {
    const window = Buffer.alloc(position.guardLength);
    const { bytesRead } = await handle.read(
      window,
      0,
      position.guardLength,
      position.resumeOffset - position.guardLength,
    );
    return bytesRead === position.guardLength && fnv1a(window) === position.guardHash;
  } catch {
    return false;
  }
}

// Match markers in one latin1 alternation; one character per byte preserves exact ASCII offsets.
export function transcriptLineGate(markers: readonly [string, ...string[]]) {
  if (markers.length === 1) {
    const marker = Buffer.from(markers[0]);
    return (line: Buffer) => line.includes(marker);
  }
  const pattern = new RegExp(
    markers.map((marker) => marker.replace(/[.*+?^${}()|[\]\\]/g, "\\

Loading Sailfork

amp;")).join("|"), ); return (line: Buffer) => pattern.test(line.toString("latin1")); } /** Returns null on read failure; caching it as empty would hide usage until size/mtime changes. */ export async function readTranscriptRecords<State>( filePath: string, format: TranscriptUsageFormat<State>, resumeFrom?: TranscriptParsePosition, options?: { readonly streamingThresholdBytes?: number }, ): Promise<TranscriptParseResult | null> { const streamingThresholdBytes = options?.streamingThresholdBytes ?? STREAMING_THRESHOLD_BYTES; const compressed = filePath.endsWith(".jsonl.zst"); let handle: NodeFSP.FileHandle; try { handle = await NodeFSP.open(filePath, "r"); } catch { return null; } try { const codec = format.state; const decodeState = codec === undefined ? undefined : Schema.decodeUnknownOption(codec.schema); const encodeState = codec === undefined ? undefined : Schema.encodeSync(codec.schema); let state = codec === undefined ? (undefined as State) : codec.initial(); let resumed = false; let start = 0; const savedState = resumeFrom === undefined || decodeState === undefined || resumeFrom.state === null ? Option.none() : decodeState(resumeFrom.state); if ( !compressed && resumeFrom !== undefined && resumeFrom.resumeOffset > 0 && (codec === undefined || Option.isSome(savedState)) && (await guardMatches(handle, resumeFrom)) ) { if (Option.isSome(savedState)) state = savedState.value; start = resumeFrom.resumeOffset; resumed = true; } const toLineString = (lineBuffer: Buffer): string => { const content = lineBuffer.length > 0 && lineBuffer[lineBuffer.length - 1] === CARRIAGE_RETURN ? lineBuffer.subarray(0, -1) : lineBuffer; return content.toString("utf8"); }; const mightCarryUsage = transcriptLineGate(format.usageMarkers); const parseLine = (lineBuffer: Buffer, lineState: State, out: UsageRecord[]): void => { if (!mightCarryUsage(lineBuffer)) return; for (const record of format.parseLine(toLineString(lineBuffer), lineState)) out.push(record); }; const records: UsageRecord[] = []; let resumeOffset = start; let scanOffset = start; let pendingChunks: Buffer[] = []; let pendingBytes = 0; let streaming: ReturnType<typeof createTranscriptJsonReader> | undefined; let decoder: NodeStringDecoder.StringDecoder | undefined; const selectPath = selectUsageFields(format.selectFields); const append = (segment: Buffer) => { if (!streaming && pendingBytes + segment.length <= streamingThresholdBytes) { if (segment.length > 0) pendingChunks.push(segment); pendingBytes += segment.length; return; } if (!streaming) { // Usage has no import-history budget; retain selected fields regardless of surrounding tool payload size. streaming = createTranscriptJsonReader(() => {}, selectPath, { maxDepth: Infinity }); decoder = new NodeStringDecoder.StringDecoder("utf8"); for (const pending of pendingChunks) streaming.write(decoder.write(pending)); pendingChunks = []; pendingBytes = 0; } streaming.write(decoder!.write(segment)); }; const finish = (lineState: State, out: UsageRecord[]) => { if (streaming) { streaming.write(decoder!.end()); const projected = streaming.finish(); for (const record of format.parseProjected(projected, lineState)) out.push(record); } else if (pendingBytes > 0) { const line = pendingChunks.length === 1 ? pendingChunks[0]! : Buffer.concat(pendingChunks, pendingBytes); parseLine(line, lineState, out); } pendingChunks = []; pendingBytes = 0; streaming = undefined; decoder = undefined; }; const input = handle.createReadStream({ start, autoClose: compressed, highWaterMark: 256 * 1024, }); // Compressed snapshots cache a full parse; decoded byte offsets cannot // resume the compressed input. const stream = compressed ? input.compose(NodeZlib.createZstdDecompress()) : input; for await (const chunk of stream) { let lineStart = 0; while (lineStart < chunk.length) { const newlineIndex = chunk.indexOf(NEWLINE, lineStart); if (newlineIndex === -1) { append(chunk.subarray(lineStart)); break; } if (!streaming && pendingBytes === 0) { parseLine(chunk.subarray(lineStart, newlineIndex), state, records); } else { append(chunk.subarray(lineStart, newlineIndex)); finish(state, records); } lineStart = newlineIndex + 1; resumeOffset = scanOffset + lineStart; } scanOffset += chunk.length; } // Parse unfinished tails against copied state so the next scan can replay from resumeOffset. const encodedState = encodeState === undefined ? null : encodeState(state); const tailRecords: UsageRecord[] = []; const tailState = decodeState === undefined ? state : Option.getOrThrow(decodeState(encodedState)); finish(tailState, tailRecords); const guardLength = compressed ? 0 : Math.min(GUARD_LENGTH, resumeOffset); let guardHash = 0; if (guardLength > 0) { const window = Buffer.alloc(guardLength); await handle.read(window, 0, guardLength, resumeOffset - guardLength); guardHash = fnv1a(window); } return { records, tailRecords, malformedRecords: format.readMalformedRecords?.(tailState) ?? 0, position: { resumeOffset: compressed ? 0 : resumeOffset, guardLength, guardHash, state: encodedState, }, resumed, }; } catch { return null; } finally { await handle.close().catch(() => undefined); } }