sailfork / t3code-github / evals/ test-signal/ run.ts

#!/usr/bin/env node
// An eval script, not server code: it shells out to git and reads files directly.
// @effect-diagnostics nodeBuiltinImport:off
import * as NodeChildProcess from "node:child_process";
import * as NodeCrypto from "node:crypto";
import * as NodeFS from "node:fs";
import * as NodeOS from "node:os";
import * as NodePath from "node:path";
import * as NodeUtil from "node:util";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http";
import { proxyConfig } from "../shared/proxy.ts";
import { cases, type Case } from "./cases.ts";
import { grade, testRange, type Comment } from "./grade.ts";

const { values } = NodeUtil.parseArgs({
  options: {
    out: { type: "string" },
    model: { type: "string", multiple: true },
    prompt: { type: "string" },
    case: { type: "string", multiple: true },
    samples: { type: "string", default: "2" },
    concurrency: { type: "string", default: "6" },
    reasoning: { type: "string", default: "low" },
  },
});
if (!values.out) throw new Error("Use --out <directory> [--model <id>] [--prompt <file>].");
const outDirectory = values.out;
const models = values.model ?? ["gpt-6.1-terra", "gpt-6-luna"];
const repo = NodePath.resolve(import.meta.dirname, "../..");
const checkFile =
  values.prompt ?? NodePath.join(repo, ".macroscope/check-run-agents/test-signal.md");
// Macroscope gives the model the body; the front matter only configures the run.
const instructions = NodeFS.readFileSync(checkFile, "utf8")
  .replace(/^---\n[\s\S]*?\n---\n/, "")
  .trim();
const words = instructions.split(/\s+/).length;
const promptHash = NodeCrypto.createHash("sha256").update(instructions).digest("hex").slice(0, 12);
const selected = values.case ? cases.filter((entry) => values.case!.includes(entry.id)) : cases;
const proxy = proxyConfig();

const git = (args: ReadonlyArray<string>) => {
  const result = NodeChildProcess.spawnSync("git", args, {
    cwd: repo,
    encoding: "utf8",
    maxBuffer: 1 << 26,
  });
  return result.status === 0 ? result.stdout : undefined;
};

/** The case's PR head as files the model can read, plus the diff it reviews. */
const world = (entry: Case) => {
  const files = new Map<string, string>();
  if (entry.type === "real") {
    const diff = git(["diff", `${entry.rev}~1`, entry.rev, "--", ...entry.paths]);
    // An empty diff would grade a guard case as a clean pass.
    if (!diff) throw new Error(`${entry.id}: no diff at ${entry.rev}.`);
    return { diff, files };
  }
  const before = git(["show", `${entry.rev}:${entry.path}`]);
  if (!before?.includes(entry.after)) throw new Error(`${entry.id}: anchor not found.`);
  const after = before.replace(entry.after, entry.after + entry.insert);
  files.set(entry.path, after);
  const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-test-signal-"));
  NodeFS.writeFileSync(NodePath.join(directory, "before"), before);
  NodeFS.writeFileSync(NodePath.join(directory, "after"), after);
  const raw = NodeChildProcess.spawnSync("git", ["diff", "--no-index", "before", "after"], {
    cwd: directory,
    encoding: "utf8",
  }).stdout;
  NodeFS.rmSync(directory, { recursive: true });
  const diff = raw
    .replace("diff --git a/before b/after", `diff --git a/${entry.path} b/${entry.path}`)
    .replace("--- a/before", `--- a/${entry.path}`)
    .replace("+++ b/after", `+++ b/${entry.path}`);
  return { diff, files };
};

// The check reads the rules at the PR head, where they always exist. Old case commits predate
// them, so serve the rules from this checkout.
const ruleFiles = new Set(["docs/internals/testing.md", "AGENTS.md"]);
const readAt = (entry: Case, files: Map<string, string>, path: string) =>
  ruleFiles.has(path)
    ? NodeFS.readFileSync(NodePath.join(repo, path), "utf8")
    : (files.get(path) ?? git(["show", `${entry.rev}:${path}`]));

/** `git grep` at `rev`, with overlay and rule files searched in their served form instead. */
const grepAt = (entry: Case, files: Map<string, string>, pattern: string, scope: string) => {
  // Status 2 is a bad pattern. Show grep's message, as a real tool would, so the model can fix it.
  const check = NodeChildProcess.spawnSync("grep", ["-E", "--", pattern], {
    input: "",
    encoding: "utf8",
  });
  if (check.status !== 0 && check.status !== 1) return check.stderr.trim() || "grep failed.";
  const served = new Map(files);
  for (const path of ruleFiles) served.set(path, readAt(entry, files, path) ?? "");
  // Served files also exist in this checkout, so git applies its own pathspec rules to the scope.
  const matched = new Set((git(["ls-files", "--", scope || "."]) ?? "").split("\n"));
  const inScope = (path: string) => matched.has(path);
  const fromRev = (git(["grep", "-n", "-E", "-e", pattern, entry.rev, "--", scope || "."]) ?? "")
    .split("\n")
    .filter((line) => line !== "")
    .map((line) => line.replace(`${entry.rev}:`, ""))
    .filter((line) => !served.has(line.slice(0, line.indexOf(":"))));
  const fromServed = [...served]
    .filter(([path]) => inScope(path))
    .flatMap(([path, text]) =>
      NodeChildProcess.spawnSync("grep", ["-n", "-E", "--", pattern], {
        input: text,
        encoding: "utf8",
      })
        .stdout.split("\n")
        .filter((line) => line !== "")
        .map((line) => `${path}:${line}`),
    );
  return [...fromServed, ...fromRev].slice(0, 80).join("\n") || "No matches.";
};

const tools = [
  {
    type: "function",
    name: "read_file",
    description: "Read a file at the pull request head, with line numbers.",
    parameters: {
      type: "object",
      properties: {
        path: { type: "string" },
        start_line: { type: "integer" },
        end_line: { type: "integer" },
      },
      required: ["path"],
    },
  },
  {
    type: "function",
    name: "grep",
    description:
      "Search the repository at the pull request head with an extended regular expression.",
    parameters: {
      type: "object",
      properties: { pattern: { type: "string" }, path: { type: "string" } },
      required: ["pattern"],
    },
  },
  {
    type: "function",
    name: "post_comment",
    description: "Post an inline review comment on a line of the pull request head.",
    parameters: {
      type: "object",
      properties: { path: { type: "string" }, line: { type: "integer" }, body: { type: "string" } },
      required: ["path", "line", "body"],
    },
  },
];

const system = (body: string) =>
  [
    "You are a Macroscope Check Run Agent reviewing a GitHub pull request.",
    "Follow the check instructions below. Read the repository at the pull request head with read_file and grep.",
    "Leave inline review comments with post_comment. Then give your final response.",
    "",
    "<check_instructions>",
    body,
    "</check_instructions>",
  ].join("\n");

const decodeResponse = Schema.decodeUnknownSync(
  Schema.fromJsonString(
    Schema.Struct({
      output: Schema.Array(Schema.Record(Schema.String, Schema.Unknown)),
      usage: Schema.optional(Schema.Struct({ total_tokens: Schema.optional(Schema.Number) })),
    }),
  ),
);
const decodeArguments = Schema.decodeUnknownSync(
  Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)),
);
const encodeRow = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));

const post = Effect.fn("evaluateTestSignal.post")(function* (body: unknown) {
  const client = (yield* HttpClient.HttpClient).pipe(
    HttpClient.retryTransient({
      retryOn: "errors-and-responses",
      schedule: Schedule.exponential("2 seconds"),
      times: 4,
    }),
  );
  const response = yield* HttpClientRequest.post(`${proxy.url}/responses`).pipe(
    HttpClientRequest.setHeaders(proxy.headers),
    HttpClientRequest.bodyJson(body),
    Effect.flatMap(client.execute),
  );
  const text = yield* response.text;
  if (response.status !== 200)
    return yield* Effect.die(new Error(`Model API ${response.status}: ${text}`));
  return decodeResponse(text);
});

const runSample = Effect.fn("evaluateTestSignal.sample")(function* (entry: Case, model: string) {
  const { diff, files } = world(entry);
  const comments: Array<Comment> = [];
  const input: Array<unknown> = [
    {
      role: "user",
      content: `Pull request diff for the files this check includes:\n\n\`\`\`diff\n${diff}\`\`\``,
    },
  ];
  let final = "";
  let tokens = 0;
  for (let step = 0; step < 16; step++) {
    const response = yield* post({
      model,
      instructions: system(instructions),
      input,
      tools,
      reasoning: { effort: values.reasoning },
      store: false,
      include: ["reasoning.encrypted_content"],
    });
    tokens += response.usage?.total_tokens ?? 0;
    input.push(...response.output);
    const calls = response.output.filter((item) => item.type === "function_call");
    final = response.output
      .filter((item) => item.type === "message")
      .flatMap((item) => (item.content as Array<{ text?: string }>).map((part) => part.text ?? ""))
      .join("");
    if (calls.length === 0) break;
    for (const call of calls) {
      const args = decodeArguments(String(call.arguments));
      let output: string;
      if (call.name === "post_comment") {
        comments.push({
          path: String(args.path),
          line: Number(args.line),
          body: String(args.body),
        });
        output = "Posted.";
      } else if (call.name === "read_file") {
        const text = readAt(entry, files, String(args.path));
        const lines = text?.split("\n") ?? [];
        const start = Math.max(1, Number(args.start_line ?? 1));
        const end = Math.min(lines.length, Number(args.end_line ?? start + 399));
        output =
          text === undefined
            ? "File not found."
            : lines
                .slice(start - 1, end)
                .map((line, index) => `${start + index}: ${line}`)
                .join("\n");
      } else {
        output = grepAt(entry, files, String(args.pattern), String(args.path ?? ""));
      }
      input.push({ type: "function_call_output", call_id: call.call_id, output });
    }
  }
  const targets = entry.targets.flatMap((target) => {
    const range = testRange(readAt(entry, files, target.path) ?? "", target.title);
    if (!range) throw new Error(`${entry.id}: target "${target.title}" not found.`);
    return [{ path: target.path, ...range }];
  });
  return { ...grade({ kind: entry.kind, targets, comments, final }), comments, final, tokens };
});

const sampleSafely = (entry: Case, model: string, sample: number) =>
  runSample(entry, model).pipe(
    Effect.map((result) => ({ case: entry.id, kind: entry.kind, model, sample, ...result })),
    Effect.catchCause((cause) =>
      Effect.succeed({ case: entry.id, kind: entry.kind, model, sample, error: String(cause) }),
    ),
  );

type Row = Effect.Success<ReturnType<typeof sampleSafely>>;
const isGraded = (row: Row) => "pass" in row;
const passed = (row: Row) => "pass" in row && row.pass;

const summary = await Effect.runPromise(
  Effect.gen(function* () {
    NodeFS.mkdirSync(outDirectory, { recursive: true });
    const samples = Number(values.samples);
    const pairs = models.flatMap((model) => selected.map((entry) => ({ model, entry })));
    const rows: Array<Row> = (yield* Effect.forEach(
      pairs.flatMap(({ model, entry }) =>
        Array.from({ length: samples }, (_, n) => ({ model, entry, n })),
      ),
      ({ model, entry, n }) => sampleSafely(entry, model, n),
      { concurrency: Number(values.concurrency) },
    )).slice();
    // A third sample settles a split.
    const splits = pairs.filter(({ model, entry }) => {
      const own = rows.filter((row) => row.model === model && row.case === entry.id);
      return own.length === 2 && own.some(passed) && !own.every(passed);
    });
    rows.push(
      ...(yield* Effect.forEach(splits, ({ model, entry }) => sampleSafely(entry, model, samples), {
        concurrency: Number(values.concurrency),
      })),
    );
    NodeFS.writeFileSync(
      NodePath.join(outDirectory, "results.ndjson"),
      rows.map((row) => encodeRow(row)).join("\n") + "\n",
    );

    const lines = [`prompt ${promptHash}, ${words} words, ${checkFile}`, ""];
    lines.push("| model | bad caught | guards clean | errors | tokens |", "|---|---|---|---|---|");
    for (const model of models) {
      const verdicts = selected.map((entry) => {
        const own = rows.filter((row) => row.model === model && row.case === entry.id);
        // A run error counts as a failed sample, so one pass and one error is not a majority.
        const graded = own.filter(isGraded);
        const passes = own.filter(passed).length;
        return {
          entry,
          pass: passes > own.length / 2,
          errors: own.length - graded.length,
        };
      });
      const count = (kind: Case["kind"]) => {
        const own = verdicts.filter((verdict) => verdict.entry.kind === kind);
        return `${own.filter((verdict) => verdict.pass).length}/${own.length}`;
      };
      const tokens = rows
        .filter((row) => row.model === model && "tokens" in row)
        .reduce((sum, row) => sum + ("tokens" in row ? row.tokens : 0), 0);
      const errors = verdicts.reduce((sum, verdict) => sum + verdict.errors, 0);
      lines.push(`| ${model} | ${count("bad")} | ${count("guard")} | ${errors} | ${tokens} |`);
      const failed = verdicts.filter((verdict) => !verdict.pass).map((verdict) => verdict.entry.id);
      if (failed.length > 0) lines.push(`|  failed: ${failed.join(", ")} | | | | |`);
    }
    const report = lines.join("\n") + "\n";
    NodeFS.writeFileSync(NodePath.join(outDirectory, "summary.md"), report);
    return report;
  }).pipe(Effect.provide(FetchHttpClient.layer)),
);
process.stdout.write(summary);