import type { ThemeAppearance } from "@t3tools/shared/themePalettes";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Queue from "effect/Queue";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import * as ChildProcess from "effect/process/ChildProcess";
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
import { publicProxy } from "./publicProxy.ts";
export class HtmlRenderBrowserError extends Schema.TaggedError<HtmlRenderBrowserError>()(
"HtmlRenderBrowserError",
{
reason: Schema.String,
cause: Schema.optional(Schema.Defect()),
output: Schema.optional(Schema.String),
exitCode: Schema.optional(Schema.Number),
signal: Schema.optional(Schema.String),
},
) {
override get message(): string {
const exit =
this.exitCode !== undefined
? ` (exit code ${this.exitCode})`
: this.signal !== undefined
? ` (signal ${this.signal})`
: "";
return `Headless Chrome could not render the page: ${this.reason}${exit}.`;
}
}
type BrowserFailure = HtmlRenderBrowserError;
export interface ConsoleMessage {
readonly level: "log" | "info" | "warning" | "error";
readonly text: string;
}
const MAX_CONSOLE_MESSAGES = 20;
const MAX_CONSOLE_TEXT_CHARS = 500;
const VIEWPORT_HEIGHT = 800;
const MAX_CAPTURE_HEIGHT = 4_000;
const CAPTURE_TIMEOUT = "20 seconds";
// Chrome blocks file access; images arrive inlined. Browser-only .localhost permits secure contexts and HTTP resources.
// publicProxy blocks local networks; pinning the main frame and forbidding popups keeps captures on the page.
const PAGE_ORIGIN = "http://t3-page.localhost";
const PAGE_URL = `${PAGE_ORIGIN}/page.html`;
const PAGE_NAME = "page.html";
const MEASURE_CONCURRENCY = 3;
const textEncoder = new TextEncoder();
// Pages reach 25 MiB; use Node's encoder.
const pageBody = (html: string) =>
Buffer.from(Buffer.from(html, "utf8").toString("base64"), "latin1");
const CdpMessage = Schema.fromJsonString(
Schema.Struct({
id: Schema.optional(Schema.Number),
method: Schema.optional(Schema.String),
sessionId: Schema.optional(Schema.String),
params: Schema.optional(Schema.Unknown),
result: Schema.optional(Schema.Unknown),
error: Schema.optional(Schema.Struct({ message: Schema.String })),
}),
);
const decodeCdpMessage = Schema.decodeUnknownOption(CdpMessage);
const encodeCdpCommand = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
const RemoteObject = Schema.Struct({
type: Schema.String,
value: Schema.optional(Schema.Unknown),
description: Schema.optional(Schema.String),
});
const decodeConsoleApiCalled = Schema.decodeUnknownOption(
Schema.Struct({ type: Schema.String, args: Schema.Array(RemoteObject) }),
);
const decodeExceptionThrown = Schema.decodeUnknownOption(
Schema.Struct({
exceptionDetails: Schema.Struct({
text: Schema.String,
exception: Schema.optional(RemoteObject),
}),
}),
);
const decodeRequestPaused = Schema.decodeUnknownOption(
Schema.Struct({
requestId: Schema.String,
request: Schema.Struct({ url: Schema.String }),
frameId: Schema.optional(Schema.String),
resourceType: Schema.optional(Schema.String),
}),
);
const decodeLogEntryAdded = Schema.decodeUnknownOption(
Schema.Struct({
entry: Schema.Struct({
level: Schema.String,
text: Schema.String,
url: Schema.optional(Schema.String),
}),
}),
);
const clipConsoleText = (text: string) => {
if (text.length <= MAX_CONSOLE_TEXT_CHARS) return text;
// Half an emoji serializes to a lone surrogate escape, which provider JSON parsers reject.
const last = text.charCodeAt(MAX_CONSOLE_TEXT_CHARS - 1);
const end =
last >= 0xd800 && last <= 0xdbff ? MAX_CONSOLE_TEXT_CHARS - 1 : MAX_CONSOLE_TEXT_CHARS;
return `${text.slice(0, end)}…`;
};
const remoteObjectText = (value: typeof RemoteObject.Type) =>
typeof value.value === "string"
? value.value
: (value.description ?? (value.value === undefined ? value.type : String(value.value)));
const CONSOLE_LEVELS: Readonly<Record<string, ConsoleMessage["level"]>> = {
log: "log",
debug: "log",
dir: "log",
dirxml: "log",
table: "log",
trace: "log",
info: "info",
warning: "warning",
error: "error",
assert: "error",
};
const consoleMessageFromEvent = (method: string, params: unknown): ConsoleMessage | undefined => {
if (method === "Runtime.consoleAPICalled") {
const event = Option.getOrUndefined(decodeConsoleApiCalled(params));
const level = event === undefined ? undefined : CONSOLE_LEVELS[event.type];
return event && level ? { level, text: event.args.map(remoteObjectText).join(" ") } : undefined;
}
if (method === "Runtime.exceptionThrown") {
const details = Option.getOrUndefined(decodeExceptionThrown(params))?.exceptionDetails;
return details
? { level: "error", text: details.exception?.description ?? details.text }
: undefined;
}
if (method === "Log.entryAdded") {
const entry = Option.getOrUndefined(decodeLogEntryAdded(params))?.entry;
return entry && (entry.level === "error" || entry.level === "warning")
? { level: entry.level, text: entry.url ? `${entry.text} ${entry.url}` : entry.text }
: undefined;
}
return undefined;
};
// Wait for fonts and two paint frames to capture late layout.
const SETTLE_EXPRESSION =
"document.fonts.ready.then(() => new Promise((resolve) => requestAnimationFrame(() => requestAnimationFrame(() => resolve(true)))))";
// scrollHeight is at least viewport height; use the root box for shorter pages.
const MEASURE_EXPRESSION =
"(() => { const root = document.documentElement; return root.scrollHeight > root.clientHeight ? root.scrollHeight : root.getBoundingClientRect().height; })()";
const Ignored = Schema.Unknown;
const Navigation = Schema.Struct({ errorText: Schema.optional(Schema.String) });
const Measured = Schema.Struct({ result: Schema.Struct({ value: Schema.Finite }) });
// Effect's spawner names the signal that ended a child only in its error message.
const EXIT_SIGNAL = /signal: '(SIG[A-Z0-9]+)'/;
interface PageEvents {
readonly mainFrameId: string;
body: Uint8Array | undefined;
loaded: Deferred.Deferred<void, BrowserFailure> | undefined;
readonly consoleMessages: Array<ConsoleMessage>;
omittedConsoleMessages: number;
}
// CDP pipe uses fd 3 in, fd 4 out and NUL-delimited JSON.
const launchBrowser = Effect.fnUntraced(function* (input: {
readonly executable: string;
readonly noSandbox: boolean;
readonly profileDirectory: string;
}) {
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
const proxy = yield* publicProxy.pipe(
Effect.mapError(
(cause) => new HtmlRenderBrowserError({ reason: "the preview proxy could not start", cause }),
),
);
const outgoing = yield* Queue.unbounded<Uint8Array>();
const child = yield* spawner
.spawn(
ChildProcess.make(
input.executable,
[
"--headless=new",
"--remote-debugging-pipe",
"--no-first-run",
"--no-default-browser-check",
"--disable-gpu",
"--hide-scrollbars",
"--mute-audio",
"--block-new-web-contents",
`--proxy-server=socks5://127.0.0.1:${proxy.port}`,
// Loopback would otherwise skip the proxy.
"--proxy-bypass-list=<-loopback>",
// WebRTC would otherwise send UDP, which no proxy carries.
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp",
...(input.noSandbox ? ["--no-sandbox"] : []),
`--user-data-dir=${input.profileDirectory}`,
"about:blank",
],
{
stdin: "ignore",
stdout: "ignore",
stderr: "pipe",
forceKillAfter: "2 seconds",
additionalFds: {
fd3: { type: "input", stream: Stream.fromQueue(outgoing) },
fd4: { type: "output" },
},
},
),
)
.pipe(
Effect.mapError(
(cause) =>
new HtmlRenderBrowserError({ reason: "the browser could not be started", cause }),
),
// Node throws some spawn errors synchronously, such as `spawn UNKNOWN`
// when policy blocks the executable; `mapError` cannot catch those.
Effect.catchDefect((cause) =>
Effect.fail(
new HtmlRenderBrowserError({ reason: "the browser could not be started", cause }),
),
),
);
let stderrTail = "";
const stderrReader = yield* child.stderr.pipe(
Stream.decodeText(),
Stream.runForEach((text) =>
Effect.sync(() => {
stderrTail = (stderrTail + text).slice(-2_048);
}),
),
Effect.ignore,
Effect.forkScoped,
);
const pending = new Map<
number,
{ readonly method: string; readonly reply: Deferred.Deferred<unknown, BrowserFailure> }
>();
const pages = new Map<string, PageEvents>();
let disconnected: BrowserFailure | undefined;
let nextId = 0;
const post = (method: string, params: Record<string, unknown>, sessionId?: string) =>
Queue.offer(
outgoing,
new TextEncoder().encode(
`${encodeCdpCommand({ id: ++nextId, method, params, sessionId })}\0`,
),
).pipe(Effect.asVoid);
// Queue shared body bytes between JSON halves to avoid copies across capture widths.
const fulfillPage = (sessionId: string, requestId: string, body: Uint8Array) =>
Queue.offerAll(outgoing, [
textEncoder.encode(
`{"id":${++nextId},"sessionId":${JSON.stringify(sessionId)},"method":"Fetch.fulfillRequest","params":{"requestId":${JSON.stringify(requestId)},"responseCode":200,"responseHeaders":[{"name":"Content-Type","value":"text/html; charset=utf-8"}],"body":"`,
),
body,
textEncoder.encode('"}}\0'),
]).pipe(Effect.asVoid);
// Incoming messages share the session span; frame acknowledgements are frequent.
const receive = Effect.fnUntraced(function* (raw: string) {
const message = Option.getOrUndefined(decodeCdpMessage(raw));
if (message?.id !== undefined) {
const waiter = pending.get(message.id);
pending.delete(message.id);
if (!waiter) return yield* Effect.void;
return yield* message.error
? Deferred.fail(
waiter.reply,
new HtmlRenderBrowserError({
reason: `${waiter.method} failed`,
cause: message.error.message,
}),
)
: Deferred.succeed(waiter.reply, message.result);
}
const sessionId = message?.sessionId;
const page = sessionId === undefined ? undefined : pages.get(sessionId);
if (sessionId === undefined || !page || !message?.method) return yield* Effect.void;
if (message.method === "Page.loadEventFired") {
return yield* page.loaded ? Deferred.succeed(page.loaded, undefined) : Effect.void;
}
if (message.method === "Fetch.requestPaused") {
const paused = Option.getOrUndefined(decodeRequestPaused(message.params));
if (!paused) return yield* Effect.void;
const body = page.body;
const url = paused.request.url.split("#", 1)[0]!;
if (url === PAGE_URL && body !== undefined) {
return yield* fulfillPage(sessionId, paused.requestId, body);
}
// Subframes may show other sites covered by Local Network Access; the main frame stays on T3's page.
const otherSiteFrame =
paused.resourceType === "Document" &&
paused.frameId !== page.mainFrameId &&
!url.startsWith(`${PAGE_ORIGIN}/`);
return yield* otherSiteFrame
? post("Fetch.continueRequest", { requestId: paused.requestId }, sessionId)
: post(
"Fetch.failRequest",
{ requestId: paused.requestId, errorReason: "AccessDenied" },
sessionId,
);
}
const consoleMessage = consoleMessageFromEvent(message.method, message.params);
if (!consoleMessage) return yield* Effect.void;
if (page.consoleMessages.length >= MAX_CONSOLE_MESSAGES) {
page.omittedConsoleMessages += 1;
return yield* Effect.void;
}
page.consoleMessages.push({
level: consoleMessage.level,
text: clipConsoleText(consoleMessage.text.replaceAll(PAGE_URL, PAGE_NAME)),
});
return yield* Effect.void;
});
// Startup aborts close the pipe; let stderr drain to recover the reason.
const disconnect = Effect.gen(function* () {
yield* Fiber.await(stderrReader).pipe(Effect.timeout("1 second"), Effect.ignore);
const exit = yield* child.exitCode.pipe(
Effect.map((code): { exitCode?: number; signal?: string; cause?: unknown } => ({
exitCode: code,
})),
Effect.catch((error) => {
const signal = EXIT_SIGNAL.exec(String(error.cause))?.[1];
return Effect.succeed(signal === undefined ? { cause: error } : { signal, cause: error });
}),
Effect.timeoutOption("1 second"),
);
const error = new HtmlRenderBrowserError({
reason: "the browser exited unexpectedly",
output: stderrTail,
...Option.getOrElse(exit, () => ({})),
});
disconnected = error;
const waiters = [...pending.values()];
pending.clear();
yield* Effect.forEach(waiters, ({ reply }) => Deferred.fail(reply, error), { discard: true });
yield* Effect.forEach(
pages.values(),
(page) => (page.loaded ? Deferred.fail(page.loaded, error) : Effect.void),
{ discard: true },
);
});
// Chunks may split messages; screenshot replies span many chunks.
let partial: Array<string> = [];
yield* child.getOutputFd(4).pipe(
Stream.decodeText(),
Stream.runForEach((text) => {
const complete: Array<string> = [];
let start = 0;
for (let end = text.indexOf("\0"); end !== -1; end = text.indexOf("\0", start)) {
partial.push(text.slice(start, end));
complete.push(partial.join(""));
partial = [];
start = end + 1;
}
if (start < text.length) partial.push(text.slice(start));
return Effect.forEach(complete, receive, { discard: true });
}),
Effect.ignore,
Effect.andThen(disconnect),
Effect.forkScoped,
);
// Signals can miss a confined browser (AppArmor in LXC), so ask it to quit first.
// Runs before the spawn's release, which still signals a browser that stays up.
yield* Effect.addFinalizer(() =>
disconnected
? Effect.void
: post("Browser.close", {}).pipe(
Effect.andThen(child.exitCode),
Effect.timeout("2 seconds"),
Effect.ignore,
),
);
const send = Effect.fn("headlessChrome.send")(function* <A>(
method: string,
params: Record<string, unknown>,
result: Schema.Decoder<A>,
sessionId?: string,
) {
if (disconnected) return yield* disconnected;
const id = ++nextId;
const reply = yield* Deferred.make<unknown, BrowserFailure>();
pending.set(id, { method, reply });
const message = encodeCdpCommand({ id, method, params, ...(sessionId ? { sessionId } : {}) });
yield* Queue.offer(outgoing, new TextEncoder().encode(`${message}\0`));
return yield* Deferred.await(reply).pipe(
Effect.flatMap(Schema.decodeUnknownEffect(result)),
Effect.mapError((error) =>
error._tag === "HtmlRenderBrowserError"
? error
: new HtmlRenderBrowserError({
reason: `${method} returned an unexpected result`,
cause: error,
}),
),
);
});
const openPage = Effect.fnUntraced(function* (width: number, appearance: ThemeAppearance) {
const { targetId } = yield* send(
"Target.createTarget",
{ url: "about:blank" },
Schema.Struct({ targetId: Schema.String }),
);
const { sessionId } = yield* send(
"Target.attachToTarget",
{ targetId, flatten: true },
Schema.Struct({ sessionId: Schema.String }),
);
const events: PageEvents = {
// A page target's id is its main frame's id.
mainFrameId: targetId,
body: undefined,
loaded: undefined,
consoleMessages: [],
omittedConsoleMessages: 0,
};
pages.set(sessionId, events);
yield* send("Page.enable", {}, Ignored, sessionId);
yield* send("Runtime.enable", {}, Ignored, sessionId);
yield* send("Log.enable", {}, Ignored, sessionId);
yield* send(
"Emulation.setEmulatedMedia",
{ features: [{ name: "prefers-color-scheme", value: appearance }] },
Ignored,
sessionId,
);
// Previews have no use for WebRTC, whose ICE servers can make the browser
// resolve names outside the proxy. Runs before page scripts in every frame.
yield* send(
"Page.addScriptToEvaluateOnNewDocument",
{
source: "delete window.RTCPeerConnection; delete window.webkitRTCPeerConnection;",
runImmediately: true,
},
Ignored,
sessionId,
);
// Intercept T3's page and all document navigations to keep the main frame on it.
yield* send(
"Fetch.enable",
{
patterns: [
{ urlPattern: `${PAGE_ORIGIN}/*` },
{ urlPattern: "*", resourceType: "Document" },
],
},
Ignored,
sessionId,
);
yield* send(
"Emulation.setDeviceMetricsOverride",
{ width, height: VIEWPORT_HEIGHT, deviceScaleFactor: 1, mobile: false },
Ignored,
sessionId,
);
const load = Effect.fnUntraced(function* (body: Uint8Array, urlFragment: string) {
const loaded = yield* Deferred.make<void, BrowserFailure>();
events.body = body;
events.loaded = loaded;
const navigation = yield* send(
"Page.navigate",
{ url: `${PAGE_URL}${urlFragment}` },
Navigation,
sessionId,
);
if (navigation.errorText) {
return yield* new HtmlRenderBrowserError({ reason: "the page could not be opened" });
}
yield* Deferred.await(loaded);
yield* send(
"Runtime.evaluate",
{ expression: SETTLE_EXPRESSION, awaitPromise: true },
Ignored,
sessionId,
);
const measured = yield* send(
"Runtime.evaluate",
{ expression: MEASURE_EXPRESSION, returnByValue: true },
Measured,
sessionId,
);
return Math.max(0, Math.ceil(measured.result.value));
});
const screenshot = (height: number) =>
send(
"Page.captureScreenshot",
{
format: "png",
captureBeyondViewport: true,
clip: { x: 0, y: 0, width, height, scale: 1 },
},
Schema.Struct({ data: Schema.String }),
sessionId,
).pipe(Effect.map(({ data }) => data));
const consoleMessages = (): ReadonlyArray<ConsoleMessage> =>
events.omittedConsoleMessages === 0
? [...events.consoleMessages]
: [
...events.consoleMessages,
{
level: "warning",
text: `${events.omittedConsoleMessages} more console messages were omitted.`,
},
];
const close = send("Target.closeTarget", { targetId }, Ignored).pipe(
Effect.ensuring(Effect.sync(() => pages.delete(sessionId))),
Effect.ignore,
);
return { load, screenshot, consoleMessages, close };
});
// Each capture opens one page per browser, so this note belongs to that page alone.
const fakeIpNote = (): ReadonlyArray<ConsoleMessage> =>
[...proxy.fakeIpHosts].map(([host, address]) => ({
level: "warning",
text: clipConsoleText(
`DNS returned blocked 198.18.0.0/15 addresses, which may come from Clash or Mihomo fake-IP DNS. Inline the resource, add its host to fake-ip-filter, or switch DNS to redir-host. Affected host: ${host} (${address})`,
),
}));
return { openPage, fakeIpNote };
});
const scratchDirectory = Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
return yield* Effect.acquireRelease(
fileSystem.makeTempDirectory({ prefix: "t3-html-preview-" }),
(directory) => fileSystem.remove(directory, { recursive: true }).pipe(Effect.ignore),
).pipe(
Effect.mapError(
(cause) =>
new HtmlRenderBrowserError({ reason: "a temporary directory could not be created", cause }),
),
);
});
export const captureHtmlScreenshot = Effect.fn("headlessChrome.captureHtmlScreenshot")(
function* (input: {
readonly executable: string;
readonly noSandbox: boolean;
readonly html: string;
readonly width: number;
readonly urlFragment: string;
readonly appearance: ThemeAppearance;
}) {
const path = yield* Path.Path;
const directory = yield* scratchDirectory;
const browser = yield* launchBrowser({
executable: input.executable,
noSandbox: input.noSandbox,
profileDirectory: path.join(directory, "profile"),
});
const page = yield* browser.openPage(input.width, input.appearance);
const contentHeight = yield* page.load(pageBody(input.html), input.urlFragment);
const capturedHeight = Math.max(1, Math.min(contentHeight, MAX_CAPTURE_HEIGHT));
const png = yield* page.screenshot(capturedHeight);
return {
png,
contentHeight,
capturedHeight,
consoleMessages: [...page.consoleMessages(), ...browser.fakeIpNote()],
};
},
Effect.scoped,
Effect.timeoutOrElse({
duration: CAPTURE_TIMEOUT,
orElse: () =>
Effect.fail(
new HtmlRenderBrowserError({
reason: `the page did not finish loading within ${CAPTURE_TIMEOUT}`,
}),
),
}),
);
/** Fresh load per width, since pages may lay out only at load. */
export const measureHtmlHeights = Effect.fn("headlessChrome.measureHtmlHeights")(function* (input: {
readonly executable: string;
readonly noSandbox: boolean;
readonly html: string;
readonly widths: ReadonlyArray<number>;
readonly urlFragment: string;
}) {
const path = yield* Path.Path;
const directory = yield* scratchDirectory;
const browser = yield* launchBrowser({
executable: input.executable,
noSandbox: input.noSandbox,
profileDirectory: path.join(directory, "profile"),
});
const body = pageBody(input.html);
return yield* Effect.forEach(
input.widths,
(width) =>
browser.openPage(width, "dark").pipe(
Effect.flatMap((page) =>
page.load(body, input.urlFragment).pipe(Effect.ensuring(page.close)),
),
Effect.map((height) => [width, height] as const),
),
{ concurrency: MEASURE_CONCURRENCY },
);
}, Effect.scoped);