sailfork / t3code-github / .repos/ alchemy-effect/ packages/ git/ src/ Jobs/ Import.ts

/**
 * The import alarm job's git client (DESIGN.md §2.3 Import).
 *
 * A minimal **smart-HTTP protocol v0 client**: fetches and parses the
 * `info/refs?service=git-upload-pack` advertisement, then POSTs a
 * `git-upload-pack` request (wants + optional `deepen`, no haves,
 * immediate `done`, **no side-band** — so the pack follows the `NAK`
 * verbatim) and returns the raw pack bytes for the Repo DO to feed
 * through the same `ingestPack` path as a push, with the same 50 MiB cap
 * (documented v1 limit: small / depth-limited imports only).
 *
 * Network I/O goes through the Effect `HttpClient` service; callers
 * provide `FetchHttpClient.layer` (see `RepoObject.ts`'s alarm handler).
 */
import * as Effect from "effect/Effect";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Schema from "effect/Schema";
import { concatBytes, isOid, ZERO_OID } from "../Protocol/ObjectCodec.ts";
import { flushPkt, pktPayloadText, pktText, readPktLineAt } from "../Protocol/Pkt.ts";

// ─────────────────────────────────────────────────────────────────────────────
// Errors & shapes
// ─────────────────────────────────────────────────────────────────────────────

/**
 * Raised when the source remote misbehaves: HTTP failures, non-200s,
 * malformed advertisements, oversize packs, or an `ERR` pkt from the
 * remote. The Repo DO maps this onto the repo's import job state.
 */
export class ImportClientError extends Schema.TaggedError<ImportClientError>()(
  "ImportClientError",
  { reason: Schema.String },
) {}

/** The import source, as accepted by `POST /api/v1/repos/import`. */
export interface ImportSource {
  /** Smart-HTTP base URL of the source repository (with or without `.git`). */
  readonly url: string;
  /** Restrict the import to a single ref (full or short name). */
  readonly ref?: string | undefined;
  /** Depth-limit the imported history (`deepen <n>`). */
  readonly depth?: number | undefined;
}

/** One ref parsed from the remote advertisement. */
export interface RemoteRef {
  readonly name: string;
  readonly oid: string;
  /** Peeled target from a `<name>^{}` line (annotated tags). */
  readonly peeled?: string | undefined;
}

/** A parsed v0 `info/refs` advertisement. */
export interface RemoteAdvertisement {
  readonly refs: ReadonlyArray<RemoteRef>;
  /**
   * The remote HEAD's symref target (e.g. `refs/heads/main`) from the
   * `symref=HEAD:...` capability, or `null` when not advertised.
   */
  readonly head: string | null;
  /** The remote's advertised capability set. */
  readonly capabilities: ReadonlySet<string>;
}

/** The result of {@link runImport}: everything the Repo DO needs to ingest. */
export interface ImportResult {
  /** The refs to create locally (already filtered by `source.ref`). */
  readonly refs: ReadonlyArray<RemoteRef>;
  /** Short default-branch name derived from the remote HEAD, or `null`. */
  readonly defaultBranch: string | null;
  /**
   * The raw pack bytes, or `undefined` when the remote is empty (nothing
   * to ingest).
   */
  readonly pack: Uint8Array | undefined;
  /** `shallow <oid>` boundary commits reported by the remote. */
  readonly shallow: ReadonlyArray<string>;
}

/** Options of {@link runImport}. */
export interface ImportOptions {
  readonly source: ImportSource;
  /**
   * Hard cap on the fetched pack size (bytes).
   * @default 52_428_800 (50 MiB, the same cap as push ingest)
   */
  readonly maxPackBytes?: number | undefined;
}

const DEFAULT_MAX_PACK = 50 * 1024 * 1024;

// ─────────────────────────────────────────────────────────────────────────────
// Advertisement
// ─────────────────────────────────────────────────────────────────────────────

const normalizeBase = (url: string): string => url.replace(/\/+$/, "");

/** Decodes all pkt-lines of a buffer into text/flush tokens. */
type Token = { readonly _tag: "line"; readonly text: string } | { readonly _tag: "flush" };

const tokenize = (buf: Uint8Array): Effect.Effect<ReadonlyArray<Token>, ImportClientError> =>
  Effect.suspend(() => {
    const out: Array<Token> = [];
    let pos = 0;
    while (pos < buf.length) {
      const r = readPktLineAt(buf, pos);
      if (r._tag === "incomplete") {
        return Effect.fail(ImportClientError.make({ reason: "truncated advertisement" }));
      }
      if (r._tag === "invalid") {
        return Effect.fail(ImportClientError.make({ reason: r.reason }));
      }
      if (r.pkt._tag === "flush") {
        out.push({ _tag: "flush" });
      } else if (r.pkt._tag === "data") {
        out.push({ _tag: "line", text: pktPayloadText(r.pkt.payload) });
      }
      pos = r.next;
    }
    return Effect.succeed(out);
  });

/**
 * Parses a v0 `git-upload-pack` advertisement body (including the
 * `# service=` prelude). Exposed for unit testing against fixture bytes.
 */
export const parseAdvertisement = Effect.fn(function* (body: Uint8Array) {
  const tokens = yield* tokenize(body);
  const refs: Array<RemoteRef> = [];
  const peeled = new Map<string, string>();
  let capabilities: ReadonlySet<string> = new Set();
  let head: string | null = null;
  let sawCaps = false;

  for (const token of tokens) {
    if (token._tag === "flush") continue;
    const text = token.text;
    if (text.startsWith("#")) continue; // "# service=git-upload-pack"
    if (text.startsWith("ERR ")) {
      return yield* ImportClientError.make({
        reason: `remote error: ${text.slice(4)}`,
      });
    }
    let line = text;
    if (!sawCaps) {
      // First ref line carries "\0cap1 cap2 ..."
      const nul = text.indexOf("\0");
      if (nul !== -1) {
        const caps = new Set(
          text
            .slice(nul + 1)
            .split(" ")
            .filter(Boolean),
        );
        capabilities = caps;
        for (const cap of caps) {
          if (cap.startsWith("symref=HEAD:")) {
            head = cap.slice("symref=HEAD:".length);
          }
        }
        line = text.slice(0, nul);
        sawCaps = true;
      }
    }
    const space = line.indexOf(" ");
    if (space !== 40 || !isOid(line.slice(0, 40))) continue;
    const oid = line.slice(0, 40);
    const name = line.slice(space + 1);
    if (oid === ZERO_OID && name === "capabilities^{}") {
      continue; // empty repo advertisement
    }
    if (name.endsWith("^{}")) {
      peeled.set(name.slice(0, -3), oid);
    } else {
      refs.push({ name, oid });
    }
  }

  const withPeeled = refs.map((ref) => {
    const p = peeled.get(ref.name);
    return p === undefined ? ref : { ...ref, peeled: p };
  });

  return {
    refs: withPeeled,
    head,
    capabilities,
  } satisfies RemoteAdvertisement;
});

/**
 * Fetches and parses the remote's `info/refs?service=git-upload-pack`
 * advertisement.
 */
export const fetchAdvertisement = Effect.fn(function* (url: string) {
  const client = yield* HttpClient.HttpClient;
  const response = yield* client
    .get(`${normalizeBase(url)}/info/refs?service=git-upload-pack`)
    .pipe(
      Effect.mapError((error) =>
        ImportClientError.make({ reason: `info/refs failed: ${error.message}` }),
      ),
    );
  if (response.status !== 200) {
    return yield* ImportClientError.make({
      reason: `info/refs returned HTTP ${response.status}`,
    });
  }
  const body = yield* response.arrayBuffer.pipe(
    Effect.mapError((error) =>
      ImportClientError.make({ reason: `info/refs body read: ${error.message}` }),
    ),
  );
  return yield* parseAdvertisement(new Uint8Array(body));
});

// ─────────────────────────────────────────────────────────────────────────────
// upload-pack POST
// ─────────────────────────────────────────────────────────────────────────────

/** Builds the v0 upload-pack request body: wants, optional deepen, done. */
export const buildUploadPackRequest = (
  wants: ReadonlyArray<string>,
  options?: { readonly depth?: number | undefined },
): Uint8Array => {
  const parts: Array<Uint8Array> = [];
  wants.forEach((oid, index) => {
    parts.push(
      index === 0
        ? pktText(`want ${oid} ofs-delta agent=alchemy-git-service/1`)
        : pktText(`want ${oid}`),
    );
  });
  if (options?.depth !== undefined) {
    parts.push(pktText(`deepen ${options.depth}`));
  }
  parts.push(flushPkt);
  parts.push(pktText("done"));
  return concatBytes(parts);
};

/**
 * The parsed upload-pack response: the shallow boundary lines (when a
 * `deepen` was requested) and the raw pack bytes that follow the `NAK`.
 */
export interface UploadPackResponse {
  readonly shallow: ReadonlyArray<string>;
  readonly pack: Uint8Array;
}

/**
 * Splits a (non-side-band) upload-pack response into its shallow section /
 * NAK and the trailing raw pack. Exposed for unit testing.
 */
export const parseUploadPackResponse = Effect.fn(function* (body: Uint8Array) {
  const shallow: Array<string> = [];
  let pos = 0;
  for (;;) {
    if (pos >= body.length) {
      return yield* ImportClientError.make({
        reason: "upload-pack response ended before NAK/ACK",
      });
    }
    const r = readPktLineAt(body, pos);
    if (r._tag === "incomplete" || r._tag === "invalid") {
      return yield* ImportClientError.make({
        reason: r._tag === "invalid" ? r.reason : "truncated upload-pack response",
      });
    }
    pos = r.next;
    if (r.pkt._tag !== "data") continue; // flush after the shallow section
    const text = pktPayloadText(r.pkt.payload);
    if (text.startsWith("shallow ")) {
      const oid = text.slice("shallow ".length);
      if (isOid(oid)) shallow.push(oid);
      continue;
    }
    if (text.startsWith("unshallow ")) continue;
    if (text.startsWith("ERR ")) {
      return yield* ImportClientError.make({
        reason: `remote error: ${text.slice(4)}`,
      });
    }
    if (text === "NAK" || text.startsWith("ACK ")) {
      // Pack bytes follow verbatim (no side-band was requested).
      return {
        shallow,
        pack: body.subarray(pos),
      } satisfies UploadPackResponse;
    }
    // Ignore anything else (progress from misbehaving servers).
  }
});

/**
 * POSTs the upload-pack request and returns the shallow lines + raw pack.
 */
export const fetchPack = Effect.fn(function* (
  url: string,
  wants: ReadonlyArray<string>,
  options?: {
    readonly depth?: number | undefined;
    readonly maxPackBytes?: number | undefined;
  },
) {
  const client = yield* HttpClient.HttpClient;
  const maxPack = options?.maxPackBytes ?? DEFAULT_MAX_PACK;
  const request = HttpClientRequest.post(`${normalizeBase(url)}/git-upload-pack`).pipe(
    HttpClientRequest.setHeaders({
      "content-type": "application/x-git-upload-pack-request",
      accept: "application/x-git-upload-pack-result",
    }),
    HttpClientRequest.bodyUint8Array(buildUploadPackRequest(wants, { depth: options?.depth })),
  );
  const response = yield* client
    .execute(request)
    .pipe(
      Effect.mapError((error) =>
        ImportClientError.make({ reason: `git-upload-pack failed: ${error.message}` }),
      ),
    );
  if (response.status !== 200) {
    return yield* ImportClientError.make({
      reason: `git-upload-pack returned HTTP ${response.status}`,
    });
  }
  const body = yield* response.arrayBuffer.pipe(
    Effect.mapError((error) =>
      ImportClientError.make({
        reason: `git-upload-pack body read: ${error.message}`,
      }),
    ),
  );
  if (body.byteLength > maxPack + 65536) {
    return yield* ImportClientError.make({
      reason: `source pack exceeds the ${maxPack} byte import cap`,
    });
  }
  const parsed = yield* parseUploadPackResponse(new Uint8Array(body));
  if (parsed.pack.byteLength > maxPack) {
    return yield* ImportClientError.make({
      reason: `source pack exceeds the ${maxPack} byte import cap`,
    });
  }
  return parsed;
});

// ─────────────────────────────────────────────────────────────────────────────
// Orchestration
// ─────────────────────────────────────────────────────────────────────────────

/** Resolves `source.ref` (full or short name) against advertised refs. */
const selectRefs = (
  advertised: ReadonlyArray<RemoteRef>,
  ref: string | undefined,
): ReadonlyArray<RemoteRef> | undefined => {
  if (ref === undefined) {
    return advertised.filter(
      (r) => r.name.startsWith("refs/heads/") || r.name.startsWith("refs/tags/"),
    );
  }
  const candidates = [ref, `refs/heads/${ref}`, `refs/tags/${ref}`];
  const match = advertised.find((r) => candidates.includes(r.name));
  return match === undefined ? undefined : [match];
};

/**
 * Runs the full import client flow: advertisement → ref selection →
 * upload-pack POST. Returns everything the Repo DO's alarm job needs to
 * ingest the history and create the refs. Requires `HttpClient.HttpClient`
 * (provide `FetchHttpClient.layer`).
 */
export const runImport = Effect.fn(function* (options: ImportOptions) {
  const { source } = options;
  const advertisement = yield* fetchAdvertisement(source.url);

  const selected = selectRefs(advertisement.refs, source.ref);
  if (selected === undefined) {
    return yield* ImportClientError.make({
      reason: `ref '${source.ref}' not found on the remote`,
    });
  }

  const defaultBranch =
    advertisement.head !== null && advertisement.head.startsWith("refs/heads/")
      ? advertisement.head.slice("refs/heads/".length)
      : null;

  if (selected.length === 0) {
    // Empty remote — nothing to fetch.
    return {
      refs: [],
      defaultBranch,
      pack: undefined,
      shallow: [],
    } satisfies ImportResult;
  }

  // Want the peeled targets too? No — wanting the tag object oid pulls the
  // whole chain; peeled lines are advisory.
  const wants = Array.from(new Set(selected.map((r) => r.oid)));
  const { shallow, pack } = yield* fetchPack(source.url, wants, {
    depth: source.depth,
    maxPackBytes: options.maxPackBytes,
  });

  return {
    refs: selected,
    defaultBranch,
    pack,
    shallow,
  } satisfies ImportResult;
});