sailfork / t3code-github / apps/ mobile/ src/ features/ diffs/ nativeReviewDiffHighlighter.ts

import { tokenizeCode, type HighlightScheme, type TokenizedCode } from "../../lib/highlights";
import type { NativeReviewDiffFile, NativeReviewDiffSource } from "./nativeReviewDiffTypes";
import type { NativeReviewDiffRow, NativeReviewDiffToken } from "./nativeReviewDiffSurface";
import { diffHighlightTraceId, traceDiffHighlight } from "./nativeReviewDiffTrace";

interface NativeReviewDiffLineRow extends NativeReviewDiffRow {
  readonly kind: "line";
  readonly fileId: string;
  readonly content: string;
}

export interface HighlightNativeReviewDiffVisibleRowsInput {
  readonly rows: ReadonlyArray<NativeReviewDiffRow>;
  readonly files: ReadonlyArray<NativeReviewDiffFile>;
  readonly scheme: HighlightScheme;
  readonly firstRowIndex: number;
  readonly lastRowIndex: number;
  readonly overscanRows?: number;
  readonly maxRows?: number;
  readonly alreadyHighlightedRowIds?: ReadonlySet<string>;
  readonly collapsedFileIds?: ReadonlySet<string>;
  readonly signal?: AbortSignal;
  readonly onChunk?: (chunk: NativeReviewDiffHighlightChunk) => void;
}

export interface NativeReviewDiffHighlightChunk {
  readonly tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>>;
  readonly retryableRowIds: ReadonlyArray<string>;
  readonly rowCount: number;
  readonly durationMs: number;
}

const NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS = 160;
const NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS = 360;
function isHighlightableLineRow(row: NativeReviewDiffRow): row is NativeReviewDiffLineRow {
  return row.kind === "line" && typeof row.fileId === "string" && typeof row.content === "string";
}

function hasConsecutiveLineNumbers(
  previous: number | null | undefined,
  next: number | null | undefined,
): boolean {
  return typeof previous === "number" && typeof next === "number" && next === previous + 1;
}

// Native lexing is non-incremental. Cache theme-independent raw records, never
// expanded token objects for an entire large file. Bound both entry count and bytes.
const MAX_TOKEN_CACHE_BYTES = 64 * 1024 * 1024;
const MAX_TOKEN_CACHE_ENTRIES = 8;
// LRU entries own only token data; source contexts and diff rows can collect independently.
const tokenCacheKeys = new WeakMap<object, object>();
const tokenCache = new Map<object, { language: string; tokens: TokenizedCode }>();
let tokenCacheBytes = 0;

function getTokenCacheKey(context: object): object {
  let key = tokenCacheKeys.get(context);
  if (!key) {
    key = {};
    tokenCacheKeys.set(context, key);
  }
  return key;
}

function getCachedTokens(
  cacheKey: object,
  contents: string | (() => string),
  language: string,
): TokenizedCode {
  const cached = tokenCache.get(cacheKey);
  if (cached) {
    tokenCache.delete(cacheKey);
    tokenCacheBytes -= cached.tokens.byteLength;
    if (cached.language === language) {
      tokenCache.set(cacheKey, cached);
      tokenCacheBytes += cached.tokens.byteLength;
      return cached.tokens;
    }
  }
  const tokens = tokenizeCode(typeof contents === "string" ? contents : contents(), language);
  if (tokens.byteLength <= MAX_TOKEN_CACHE_BYTES) {
    tokenCache.set(cacheKey, { language, tokens });
    tokenCacheBytes += tokens.byteLength;
  }
  while (tokenCacheBytes > MAX_TOKEN_CACHE_BYTES || tokenCache.size > MAX_TOKEN_CACHE_ENTRIES) {
    const oldest = tokenCache.keys().next().value!;
    tokenCacheBytes -= tokenCache.get(oldest)!.tokens.byteLength;
    tokenCache.delete(oldest);
  }
  return tokens;
}

const fullSourceContexts = new WeakMap<
  NativeReviewDiffSource,
  { old: FullSourceContext; new: FullSourceContext }
>();

interface FullSourceContext {
  readonly contents: string;
  readonly comparisonLines: ReadonlyArray<string>;
  readonly validation: WeakMap<ReadonlyArray<NativeReviewDiffLineRow>, boolean>;
}

const sourceRowsByFileCache = new WeakMap<
  ReadonlyArray<NativeReviewDiffRow>,
  Map<string, NativeReviewDiffLineRow[]>
>();

function getSourceRowsByFile(rows: ReadonlyArray<NativeReviewDiffRow>) {
  let byFile = sourceRowsByFileCache.get(rows);
  if (!byFile) {
    byFile = new Map();
    for (const row of rows) {
      if (!isHighlightableLineRow(row)) continue;
      const fileRows = byFile.get(row.fileId);
      if (fileRows) fileRows.push(row);
      else byFile.set(row.fileId, [row]);
    }
    sourceRowsByFileCache.set(rows, byFile);
  }
  return byFile;
}

function matchesSourceSide(
  context: FullSourceContext,
  rows: ReadonlyArray<NativeReviewDiffLineRow>,
  side: DiffSide,
) {
  const cached = context.validation.get(rows);
  if (cached !== undefined) return cached;
  // Context participates on both sides, even though it displays NEW tokens.
  // A known mismatch anywhere invalidates grammar from the entire source side.
  const matches = rows.every((row) => {
    if (row.change === (side === "old" ? "add" : "delete")) return true;
    const lineNumber = side === "old" ? row.oldLineNumber : row.newLineNumber;
    return (
      typeof lineNumber === "number" && context.comparisonLines[lineNumber - 1] === row.content
    );
  });
  context.validation.set(rows, matches);
  return matches;
}

function createFullSourceContext(contents: string): FullSourceContext {
  return {
    contents,
    // The diff parser retains CR on patch rows, unlike the token output.
    comparisonLines: contents.split("\n"),
    validation: new WeakMap(),
  };
}

function getFullSourceContexts(source: NativeReviewDiffSource) {
  let contexts = fullSourceContexts.get(source);
  if (!contexts) {
    contexts = {
      old: createFullSourceContext(source.oldContents),
      new: createFullSourceContext(source.newContents),
    };
    fullSourceContexts.set(source, contexts);
  }
  return contexts;
}

type DiffSide = "old" | "new";

interface SourceSegment {
  readonly fileId: string;
  readonly side: DiffSide;
  readonly rows: NativeReviewDiffLineRow[];
}

// Row arrays are immutable snapshots; comments and hidden hunks retain source context.
const sourceSegmentsCache = new WeakMap<
  ReadonlyArray<NativeReviewDiffRow>,
  Map<string, { readonly segment: SourceSegment; readonly index: number }>
>();

function getSourceSegments(rows: ReadonlyArray<NativeReviewDiffRow>) {
  const cached = sourceSegmentsCache.get(rows);
  if (cached) return cached;

  const segmentsByRowId = new Map<
    string,
    { readonly segment: SourceSegment; readonly index: number }
  >();
  for (const side of ["old", "new"] as const) {
    let segment: SourceSegment | undefined;
    for (const row of rows) {
      if (row.kind === "comment" || row.kind === "hunk") continue;
      if (!isHighlightableLineRow(row)) {
        segment = undefined;
        continue;
      }
      if (row.change === (side === "old" ? "add" : "delete")) continue;

      const previous = segment?.rows.at(-1);
      const lineNumber = side === "old" ? row.oldLineNumber : row.newLineNumber;
      const previousLineNumber = side === "old" ? previous?.oldLineNumber : previous?.newLineNumber;
      if (
        !segment ||
        segment.fileId !== row.fileId ||
        !hasConsecutiveLineNumbers(previousLineNumber, lineNumber)
      ) {
        // A gap has no known incoming grammar state. Highlight the available
        // fragment independently; never join it to an earlier, incomplete hunk.
        segment = { fileId: row.fileId, side, rows: [] };
      }
      if (displaySide(row) === side) {
        segmentsByRowId.set(row.id, { segment, index: segment.rows.length });
      }
      segment.rows.push(row);
    }
  }
  sourceSegmentsCache.set(rows, segmentsByRowId);
  return segmentsByRowId;
}

function displaySide(row: NativeReviewDiffLineRow): DiffSide {
  // Unified context rows have one token array. Match the NEW side, while OLD
  // context still advances deletion grammar independently.
  return row.change === "delete" ? "old" : "new";
}

function createFileMap(files: ReadonlyArray<NativeReviewDiffFile>) {
  return new Map(files.map((file) => [file.id, file]));
}

function clampRowIndex(index: number, rows: ReadonlyArray<NativeReviewDiffRow>) {
  if (rows.length === 0) {
    return 0;
  }

  return Math.min(rows.length - 1, Math.max(0, Math.floor(index)));
}

function makePlainTokenFallback(
  row: NativeReviewDiffLineRow,
): ReadonlyArray<NativeReviewDiffToken> {
  return [{ content: row.content || " ", color: null, fontStyle: null }];
}

export async function highlightNativeReviewDiffVisibleRows(
  input: HighlightNativeReviewDiffVisibleRowsInput,
): Promise<{
  readonly tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>>;
  // Deliver plain tokens for these rows without marking them permanently highlighted.
  readonly retryableRowIds: ReadonlyArray<string>;
  readonly rowCount: number;
  readonly durationMs: number;
}> {
  const request = diffHighlightTraceId(input);
  traceDiffHighlight("highlight-start", {
    request,
    first: input.firstRowIndex,
    last: input.lastRowIndex,
    overscan: input.overscanRows ?? NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS,
  });
  if (input.signal?.aborted) {
    return {
      tokensByRowId: {},
      retryableRowIds: [],
      rowCount: 0,
      durationMs: 0,
    };
  }

  const startedAt = performance.now();
  const fileMap = createFileMap(input.files);
  const overscanRows = input.overscanRows ?? NATIVE_REVIEW_DIFF_VISIBLE_OVERSCAN_ROWS;
  const maxRows = input.maxRows ?? NATIVE_REVIEW_DIFF_VISIBLE_MAX_ROWS;
  const startIndex = clampRowIndex(input.firstRowIndex - overscanRows, input.rows);
  const endIndex = clampRowIndex(input.lastRowIndex + overscanRows, input.rows);
  const visibleStartIndex = clampRowIndex(input.firstRowIndex, input.rows);
  const visibleEndIndex = clampRowIndex(input.lastRowIndex, input.rows);
  const visibleFileIds = new Set<string>();
  // Overscan warms visible files without loading unrelated neighboring sources.
  // Headers and already-highlighted rows still identify files in the viewport.
  for (let rowIndex = visibleStartIndex; rowIndex <= visibleEndIndex; rowIndex += 1) {
    const fileId = input.rows[rowIndex]?.fileId;
    if (typeof fileId === "string") visibleFileIds.add(fileId);
  }
  const selectedRows: NativeReviewDiffLineRow[] = [];

  for (
    let rowIndex = startIndex;
    rowIndex <= endIndex && selectedRows.length < maxRows;
    rowIndex += 1
  ) {
    const row = input.rows[rowIndex];
    if (
      row &&
      isHighlightableLineRow(row) &&
      visibleFileIds.has(row.fileId) &&
      !input.collapsedFileIds?.has(row.fileId) &&
      !input.alreadyHighlightedRowIds?.has(row.id) &&
      fileMap.has(row.fileId)
    ) {
      selectedRows.push(row);
    }
  }

  const tokensByRowId: Record<string, ReadonlyArray<NativeReviewDiffToken>> = {};
  const retryableRowIds: string[] = [];
  const selectedRowIds = new Set(selectedRows.map((row) => row.id));
  const tokenize = (context: object, contents: string | (() => string), language: string) => {
    const cacheKey = getTokenCacheKey(context);
    const cacheHit = tokenCache.get(cacheKey)?.language === language;
    const cached = getCachedTokens(cacheKey, contents, language);
    traceDiffHighlight("tokenized", {
      request,
      characters: typeof contents === "string" ? contents.length : undefined,
      cacheHit,
      retainedBytes: cached.byteLength,
      lines: cached.lineCount,
      aborted: input.signal?.aborted ?? false,
    });
    return cached;
  };

  const sourceFiles = new Map<string, NativeReviewDiffLineRow[]>();
  for (const row of selectedRows) {
    if (!fileMap.get(row.fileId)?.loadContents) continue;
    const fileRows = sourceFiles.get(row.fileId);
    if (fileRows) fileRows.push(row);
    else sourceFiles.set(row.fileId, [row]);
  }
  const emitFileChunk = (fileRows: ReadonlyArray<NativeReviewDiffLineRow>) => {
    if (!input.onChunk || input.signal?.aborted) return;
    const rowIds = new Set(fileRows.map((row) => row.id));
    input.onChunk({
      tokensByRowId: Object.fromEntries(fileRows.map((row) => [row.id, tokensByRowId[row.id]!])),
      retryableRowIds: retryableRowIds.filter((id) => rowIds.has(id)),
      rowCount: fileRows.length,
      durationMs: Math.round(performance.now() - startedAt),
    });
  };
  // The file filling the viewport should not wait for a sliver of its neighbor.
  for (const [fileId, fileRows] of [...sourceFiles].sort((a, b) => b[1].length - a[1].length)) {
    if (input.signal?.aborted) break;
    const file = fileMap.get(fileId)!;
    const fileTraceId = diffHighlightTraceId(fileId);
    traceDiffHighlight("highlight-source", { request, file: fileTraceId, rows: fileRows.length });
    let source: NativeReviewDiffSource | null;
    try {
      source = await file.loadContents!(input.signal);
    } catch {
      traceDiffHighlight("highlight-source-error", { request, file: fileTraceId });
      for (const row of fileRows) {
        selectedRowIds.delete(row.id);
        tokensByRowId[row.id] = makePlainTokenFallback(row);
        retryableRowIds.push(row.id);
      }
      emitFileChunk(fileRows);
      continue;
    }
    if (input.signal?.aborted) break;
    const contexts = source ? getFullSourceContexts(source) : null;
    for (const side of ["old", "new"] as const) {
      if (input.signal?.aborted) break;
      const candidate = contexts?.[side];
      const context =
        candidate &&
        matchesSourceSide(candidate, getSourceRowsByFile(input.rows).get(fileId)!, side)
          ? candidate
          : null;
      traceDiffHighlight("source-validation", {
        request,
        file: fileTraceId,
        side,
        present: !!candidate,
        matches: !!context,
        knownLines: context?.comparisonLines.length,
      });
      const matchingRows: { row: NativeReviewDiffLineRow; lineIndex: number }[] = [];
      for (const row of fileRows) {
        if (displaySide(row) !== side) continue;
        selectedRowIds.delete(row.id);
        const lineNumber = side === "old" ? row.oldLineNumber : row.newLineNumber;
        const lineIndex = typeof lineNumber === "number" ? lineNumber - 1 : -1;
        if (
          context &&
          lineIndex >= 0 &&
          lineIndex < context.comparisonLines.length &&
          context.comparisonLines[lineIndex] === row.content
        ) {
          matchingRows.push({ row, lineIndex });
        } else {
          // A changed revision or missing source cannot
          // supply trustworthy grammar. Do not cache guessed syntax colors.
          tokensByRowId[row.id] = makePlainTokenFallback(row);
        }
      }
      if (!context || matchingRows.length === 0) continue;
      const tokens = tokenize(context, context.contents, file.language);
      for (const { row, lineIndex } of matchingRows) {
        tokensByRowId[row.id] =
          tokens.highlightLine(lineIndex, input.scheme) ?? makePlainTokenFallback(row);
      }
    }
    emitFileChunk(fileRows);
    await new Promise<void>((resolve) => setTimeout(resolve, 0));
  }

  const selectedSegments = new Map<
    SourceSegment,
    { firstSelected: number; lastSelected: number }
  >();
  if (selectedRowIds.size > 0 && !input.signal?.aborted) {
    const sourceSegments = getSourceSegments(input.rows);
    for (const row of selectedRows) {
      if (!selectedRowIds.has(row.id)) continue;
      const source = sourceSegments.get(row.id);
      if (!source) continue;
      const selection = selectedSegments.get(source.segment);
      if (selection) {
        selection.lastSelected = source.index;
      } else {
        selectedSegments.set(source.segment, {
          firstSelected: source.index,
          lastSelected: source.index,
        });
      }
    }
  }

  for (const [segment, { firstSelected, lastSelected }] of selectedSegments) {
    if (input.signal?.aborted) break;
    const file = fileMap.get(segment.fileId);
    if (!file) continue;

    const tokens = tokenize(
      segment,
      () => segment.rows.map((row) => row.content).join("\n"),
      file.language,
    );
    for (let index = firstSelected; index <= lastSelected; index += 1) {
      const row = segment.rows[index]!;
      if (selectedRowIds.has(row.id) && displaySide(row) === segment.side) {
        tokensByRowId[row.id] =
          tokens.highlightLine(index, input.scheme) ?? makePlainTokenFallback(row);
      }
    }
  }

  if (input.signal?.aborted) {
    traceDiffHighlight("highlight-aborted", { request });
    return {
      tokensByRowId: {},
      retryableRowIds: [],
      rowCount: 0,
      durationMs: 0,
    };
  }

  traceDiffHighlight("highlight-result", {
    request,
    selected: selectedRows.length,
    delivered: Object.keys(tokensByRowId).length,
    retryable: retryableRowIds.length,
    colored: Object.values(tokensByRowId).filter((tokens) =>
      tokens.some((token) => token.color !== null),
    ).length,
  });
  return {
    tokensByRowId,
    retryableRowIds,
    rowCount: selectedRows.length,
    durationMs: Math.round(performance.now() - startedAt),
  };
}