// @effect-diagnostics nodeBuiltinImport:off -- isolated CLI worker; no server runtime is loaded here.
import * as NodeFSP from "node:fs/promises";
import * as NodePath from "node:path";
import * as NodeUtil from "node:util";
import { characterEntities } from "character-entities";
import { SaxesParser } from "saxes";
import * as Yauzl from "yauzl";
const MAX_INPUT = 16 * 1024 * 1024;
const MAX_EXPANDED = 32 * 1024 * 1024;
const MAX_OUTPUT = 1_000_000;
const CELL_START = Symbol("cell-start");
const CELL_END = Symbol("cell-end");
type TextPart = string | typeof CELL_START | typeof CELL_END | { fallback: string };
const TRUNCATED = "\n\n[Extracted text truncated at document extraction limits.]";
// Parts the document never displays: tracked deletions and moves, field codes, phonetic guides,
// paragraph properties such as tab-stop definitions.
const HIDDEN = new Set([
"script",
"style",
"binary-data",
"rPh",
"delText",
"moveFrom",
"instrText",
"delInstrText",
"tracked-changes",
"pPr",
]);
// @effect-diagnostics-next-line extendsNativeError:off - this worker loads no Effect runtime; Truncation is a local throw sentinel.
class Truncation extends Error {}
interface XmlNode {
name: string;
attrs: Record<string, string>;
children: Array<XmlNode | string>;
spaceCount?: number;
hasText?: boolean;
hasTextChoice?: boolean;
}
function xml(source: string, odfSpaces = false, htmlEntities = false): XmlNode {
const root: XmlNode = { name: "root", attrs: {}, children: [] };
const stack = [root];
let nodes = 0;
const parser = new SaxesParser();
if (htmlEntities) Object.assign(parser.ENTITIES, characterEntities);
parser.on("opentag", (tag) => {
if (++nodes > 200_000 || stack.length > 128) throw new Truncation("XML structure limit");
const node: XmlNode = {
name: tag.name.split(":").at(-1)!,
attrs: Object.fromEntries(
Object.entries(tag.attributes).flatMap(([key, value]) => [
[key, String(value)],
[key.split(":").at(-1)!, String(value)],
]),
),
children: [],
};
if (odfSpaces && node.name === "s") node.spaceCount = repeatCount(node.attrs.c);
stack.at(-1)!.children.push(node);
stack.push(node);
});
parser.on("closetag", () => {
const node = stack.pop()!;
const parent = stack.at(-1)!;
// Markup-compatibility fallbacks repeat their mc:Choice content, e.g. Word text boxes.
// Keep one when no Choice carries text, such as a chart with a VML caption.
if (node.name === "Fallback" && parent.hasTextChoice) {
parent.children.pop();
return;
}
if (!node.hasText) return;
parent.hasText = true;
if (node.name === "Choice") parent.hasTextChoice = true;
});
const text = (value: string) => {
const node = stack.at(-1)!;
node.children.push(value);
if (value.trim() !== "") node.hasText = true;
};
parser.on("text", text);
parser.on("cdata", text);
parser.write(source).close();
return root;
}
const children = (node: XmlNode) =>
node.children.filter((child): child is XmlNode => typeof child !== "string");
function* all(node: XmlNode, name: string | readonly string[]): Generator<XmlNode, void> {
for (const child of children(node)) {
if (typeof name === "string" ? child.name === name : name.includes(child.name)) yield child;
else yield* all(child, name);
}
}
function* text(node: XmlNode): Generator<string, void> {
if (HIDDEN.has(node.name)) return;
if (["tab", "cell"].includes(node.name)) yield "\t";
else if (["br", "line-break", "cr"].includes(node.name)) yield "\n";
else if (node.spaceCount !== undefined) {
yield " ".repeat(Math.min(MAX_OUTPUT, node.spaceCount));
if (node.spaceCount > MAX_OUTPUT) throw new Truncation("Space repeat limit");
} else
for (const child of node.children) {
if (typeof child === "string") yield child;
else yield* text(child);
}
}
function* paragraphs(node: XmlNode): Generator<TextPart, void> {
if (HIDDEN.has(node.name)) return;
for (const child of node.children) {
if (typeof child === "string") yield child;
else if (["br", "line-break", "cr"].includes(child.name)) yield "\n";
else if (["tbl", "table"].includes(child.name)) yield* table(child);
else if (["p", "h"].includes(child.name) || /^h[1-6]$/.test(child.name)) {
const style = all(child, "pStyle").next().value?.attrs.val ?? "";
const level =
child.name === "h"
? Number(child.attrs["outline-level"]) || 1
: /^h[1-6]$/.test(child.name)
? Number(child.name[1])
: Number(style.match(/^Heading([1-6])$/i)?.[1]) || 0;
if (level) yield `${"#".repeat(Math.min(6, level))} `;
yield* text(child);
yield "\n";
} else {
yield* paragraphs(child);
if (["li", "div", "section"].includes(child.name)) yield "\n";
}
}
}
function* inCell(parts: Iterable<TextPart>): Generator<TextPart, void> {
yield CELL_START;
yield* parts;
yield CELL_END;
}
function repeatCount(value: string | undefined): number {
if (value === undefined) return 1;
if (!/^[0-9]+$/.test(value) || Number(value) < 1) throw new Error("Invalid repeat count");
return Number(value);
}
function odfValue(cell: XmlNode): string {
const type = cell.attrs["office:value-type"] ?? cell.attrs["value-type"];
const attr = (name: string) => cell.attrs[`office:${name}`] ?? cell.attrs[name] ?? "";
switch (type) {
case "void":
return "";
case "string":
return attr("string-value");
case "boolean":
return attr("boolean-value");
case "date":
return attr("date-value");
case "time":
return attr("time-value");
case "currency":
return attr("value").trim() ? `${attr("value")} ${attr("currency")}`.trim() : "";
case "percentage":
return attr("value").trim() ? `${attr("value")} [ODF value-type=percentage]` : "";
default:
return attr("value");
}
}
function hasCellText(cell: XmlNode): boolean {
if (odfValue(cell).trim()) return true;
// Space markers can have enormous repeat counts; emptiness never requires expanding them.
if (HIDDEN.has(cell.name) || cell.spaceCount !== undefined) return false;
return cell.children.some((child) =>
typeof child === "string" ? !!child.trim() : hasCellText(child),
);
}
function* table(node: XmlNode): Generator<TextPart, void> {
let rowCount = 0;
for (const row of all(node, ["tr", "table-row"])) {
const cells = children(row).filter((cell) =>
["tc", "table-cell", "covered-table-cell", "td", "th"].includes(cell.name),
);
// ODS writers pad sheets with repeated empty rows and trailing cells.
while (cells.length && !hasCellText(cells.at(-1)!)) cells.pop();
if (!cells.length) continue;
const repeat = repeatCount(row.attrs["number-rows-repeated"]);
for (let index = 0; index < repeat; index++) {
if (++rowCount > 10_000) throw new Truncation("Row limit");
let column = 0;
for (const cell of cells) {
const count = repeatCount(cell.attrs["number-columns-repeated"]);
for (let i = 0; i < count; i++) {
if (column >= 512) throw new Truncation("Column limit");
if (column++) yield "\t";
yield CELL_START;
yield* paragraphs(cell);
yield { fallback: odfValue(cell) };
yield CELL_END;
}
}
yield "\n";
}
}
}
async function archive(path: string): Promise<Map<string, string>> {
const zip = await new Promise<Yauzl.ZipFile>((resolve, reject) =>
Yauzl.open(
path,
{ lazyEntries: true, validateEntrySizes: true, strictFileNames: true },
(error, result) => (error || !result ? reject(error) : resolve(result)),
),
);
const files = new Map<string, string>();
let expanded = 0;
let count = 0;
try {
await new Promise<void>((resolve, reject) => {
zip.on("error", reject);
zip.on("end", resolve);
zip.on("entry", (entry: Yauzl.Entry) => {
if (++count > 2048 || zip.entryCount > 2048) {
reject(new Truncation("Archive limit"));
return;
}
if (entry.generalPurposeBitFlag & 1) {
reject(new Error("Encrypted archive"));
return;
}
if (!/\.(xml|rels|xhtml|html|htm|opf|svg)$/i.test(entry.fileName)) {
zip.readEntry();
return;
}
if (
entry.uncompressedSize > MAX_EXPANDED ||
entry.uncompressedSize > Math.max(1024, entry.compressedSize * 200)
) {
reject(new Truncation("Archive limit"));
return;
}
expanded += entry.uncompressedSize;
if (expanded > MAX_EXPANDED) {
reject(new Truncation("Expanded byte limit"));
return;
}
zip.openReadStream(entry, (error, stream) => {
if (error || !stream) {
reject(error);
return;
}
const buffers: Buffer[] = [];
let size = 0;
stream.on("error", reject);
stream.on("data", (chunk: Buffer) => {
size += chunk.length;
if (size > entry.uncompressedSize || size > MAX_EXPANDED) {
stream.destroy(new Truncation("Entry byte limit"));
return;
}
buffers.push(chunk);
});
stream.on("end", () => {
const data = Buffer.concat(buffers);
// XML's BOM and initial '<' bytes distinguish both UTF-16 byte orders.
const encoding =
(data[0] === 0xff && data[1] === 0xfe) || (data[0] === 0x3c && data[1] === 0)
? "utf-16le"
: (data[0] === 0xfe && data[1] === 0xff) || (data[0] === 0 && data[1] === 0x3c)
? "utf-16be"
: "utf-8";
files.set(entry.fileName, new TextDecoder(encoding).decode(data));
zip.readEntry();
});
});
});
zip.readEntry();
});
} finally {
zip.close();
}
return files;
}
function* notebook(source: string): Generator<string, void> {
const book: unknown = JSON.parse(source);
if (!book || typeof book !== "object" || !("cells" in book) || !Array.isArray(book.cells))
throw new Error("Invalid notebook");
function* lines(value: unknown): Generator<string, void> {
if (typeof value === "string") yield value;
else if (Array.isArray(value))
for (const line of value) {
if (typeof line === "string") yield line;
}
}
for (const [index, cell] of book.cells.entries()) {
if (index >= 10_000) throw new Truncation("Notebook cell limit");
if (!cell || typeof cell !== "object") continue;
const kind = "cell_type" in cell && cell.cell_type === "code" ? "code" : "markdown";
yield `## Cell ${index + 1} (${kind})\n`;
if ("source" in cell) yield* lines(cell.source);
if ("outputs" in cell && Array.isArray(cell.outputs)) {
yield "\nOutput:\n";
for (const output of cell.outputs) {
if (!output || typeof output !== "object") continue;
if ("text" in output) yield* lines(output.text);
else if (
"data" in output &&
output.data &&
typeof output.data === "object" &&
"text/plain" in output.data
)
yield* lines(output.data["text/plain"]);
else if ("output_type" in output && output.output_type === "error") {
if ("ename" in output) yield* lines(output.ename);
if ("evalue" in output) {
yield ": ";
yield* lines(output.evalue);
}
yield "\n";
if ("traceback" in output)
for (const line of lines(output.traceback)) {
yield NodeUtil.stripVTControlCharacters(line);
yield "\n";
}
}
yield "\n";
}
}
yield "\n\n";
}
}
function* rtf(source: string): Generator<string, void> {
if (!source.startsWith("{\\rtf")) throw new Error("Invalid RTF document");
let state = { skip: false, uc: 1, decoder: new TextDecoder("windows-1252") };
const stack: Array<typeof state> = [];
const symbols = new Map([
["endash", "–"],
["emdash", characterEntities.mdash],
["enspace", "\u2002"],
["emspace", "\u2003"],
["qmspace", "\u2005"],
["lquote", "‘"],
["rquote", "’"],
["ldblquote", "“"],
["rdblquote", "”"],
["bullet", "•"],
["~", "\u00a0"],
["_", "\u2011"],
["-", ""],
]);
let fallback = 0;
for (let i = 0; i < source.length; i++) {
const char = source[i]!;
if (char === "{" || char === "}") fallback = 0;
if (char === "{") {
stack.push({ ...state });
if (stack.length > 128) throw new Truncation("RTF depth limit");
} else if (char === "}") state = stack.pop() ?? state;
else if (char === "\\") {
const rest = source.slice(i + 1, i + 80);
const hex = /^'([0-9a-f]{2})/i.exec(rest);
const word = /^([a-z]+)(-?\d+)? ?/i.exec(rest);
if (hex) {
if (fallback) fallback--;
else if (!state.skip)
yield state.decoder.decode(Uint8Array.of(Number.parseInt(hex[1]!, 16)));
i += 3;
} else if (word) {
i += word[0].length;
const control = word[1]!;
const value = Number(word[2]);
if (control === "bin") {
if (!Number.isSafeInteger(value) || value < 0)
throw new Error("Invalid RTF binary length");
i += value;
}
if (fallback) {
fallback--;
continue;
}
if (
[
"fonttbl",
"colortbl",
"stylesheet",
"info",
"pict",
"object",
"fldinst",
"header",
"footer",
].includes(control)
)
state.skip = true;
if (control === "ansicpg") {
// Per-byte decoding only supports single-byte Windows code pages.
if (value !== 874 && (value < 1250 || value > 1258))
throw new Error("Unsupported RTF code page");
state.decoder = new TextDecoder(`windows-${value}`, { fatal: true });
} else if (control === "uc") {
if (!Number.isSafeInteger(value) || value < 0)
throw new Error("Invalid RTF fallback count");
state.uc = value;
} else if (control === "u" && !state.skip) {
yield String.fromCharCode(value < 0 ? value + 65536 : value);
fallback = state.uc;
} else if (!state.skip && ["par", "line", "row"].includes(control)) yield "\n";
else if (!state.skip && ["tab", "cell"].includes(control)) yield "\t";
else if (!state.skip && symbols.has(control)) yield symbols.get(control)!;
} else {
i++;
if (fallback) fallback--;
else if (rest[0] === "*") state.skip = true;
// A backslash before a line break is \par; TextEdit ends paragraphs this way.
else if (!state.skip && (rest[0] === "\n" || rest[0] === "\r")) yield "\n";
else if (!state.skip && ["\\", "{", "}"].includes(rest[0]!)) yield rest[0]!;
else if (!state.skip && symbols.has(rest[0]!)) yield symbols.get(rest[0]!)!;
}
} else if (!state.skip && char !== "\n" && char !== "\r") {
if (fallback) fallback--;
else
yield char.charCodeAt(0) < 128
? char
: state.decoder.decode(Uint8Array.of(char.charCodeAt(0)));
}
}
}
async function* document(path: string, extension: string): AsyncGenerator<TextPart, void> {
if ((await NodeFSP.stat(path)).size > MAX_INPUT) throw new Truncation("Input byte limit");
if (extension === ".ipynb") yield* notebook(await NodeFSP.readFile(path, "utf8"));
else if (extension === ".rtf") yield* rtf((await NodeFSP.readFile(path)).toString("latin1"));
else {
const files = await archive(path);
const notes: string[] = [];
const skip = (name: string) =>
notes.push(`\n[Skipped document part: ${name} (missing or invalid XML).]\n`);
const read = (name: string): XmlNode => {
try {
const data = files.get(name);
if (!data) throw new Error("Missing document part");
return xml(data, [".odt", ".ods", ".odp"].includes(extension), extension === ".epub");
} catch (error) {
if (error instanceof Truncation) throw error;
skip(name);
return { name: "root", attrs: {}, children: [] };
}
};
const relations = (name: string) =>
new Map(
[...all(read(name), "Relationship")]
.filter((rel) => rel.attrs.TargetMode !== "External")
.map((rel) => [rel.attrs.Id, rel.attrs.Target]),
);
if (extension === ".docx") yield* paragraphs(read("word/document.xml"));
else if ([".odt", ".ods", ".odp"].includes(extension)) {
const content = read("content.xml");
if (extension === ".odt") yield* paragraphs(content);
else {
let index = 0;
for (const part of all(content, extension === ".ods" ? "table" : "page")) {
yield extension === ".ods"
? `\n\n## Sheet: ${part.attrs.name ?? "Untitled"}\n`
: `\n\n## Slide ${++index}\n`;
if (extension === ".ods") yield* table(part);
else yield* paragraphs(part);
}
}
} else if (extension === ".xlsx") {
const workbook = read("xl/workbook.xml");
const dateSystem = ["1", "true"].includes(
all(workbook, "workbookPr").next().value?.attrs.date1904 ?? "",
)
? "1904"
: "1900";
const styles = files.has("xl/styles.xml") ? read("xl/styles.xml") : undefined;
const customFormats = new Map(
styles
? [...all(styles, "numFmt")].map((node) => [node.attrs.numFmtId, node.attrs.formatCode])
: [],
);
const cellFormats = styles ? all(styles, "cellXfs").next().value : undefined;
const formats = cellFormats ? children(cellFormats) : [];
const strings = files.has("xl/sharedStrings.xml")
? [...all(read("xl/sharedStrings.xml"), "si")]
: [];
const populatedStrings = strings.map(hasCellText);
const rels = relations("xl/_rels/workbook.xml.rels");
for (const sheet of all(workbook, "sheet")) {
yield `\n## Sheet: ${sheet.attrs.name}\n`;
const target = rels.get(sheet.attrs["r:id"] ?? sheet.attrs.id);
if (!target) {
skip(`sheet relationship ${sheet.attrs.name}`);
continue;
}
const name = target.startsWith("/")
? target.slice(1)
: NodePath.posix.normalize(`xl/${target}`);
const populated = (cell: XmlNode) => {
const value = all(cell, "v").next().value;
if (cell.attrs.t === "s") {
const raw = value?.children.find((child) => typeof child === "string")?.trim();
return !!raw && !!populatedStrings[Number(raw)];
}
if (cell.attrs.t === "inlineStr") return hasCellText(cell);
if (value && hasCellText(value)) return true;
for (const formula of all(cell, "f")) if (hasCellText(formula)) return true;
return false;
};
let rowCount = 0;
for (const row of all(read(name), "row")) {
const cells: Array<XmlNode | undefined> = [];
let nextColumn = 0;
let columnsTruncated = false;
for (const cell of children(row).filter((node) => node.name === "c")) {
const letters = cell.attrs.r?.match(/^[A-Z]+/)?.[0];
const column = letters
? [...letters].reduce((n, char) => n * 26 + char.charCodeAt(0) - 64, 0) - 1
: nextColumn;
nextColumn = column + 1;
if (!populated(cell)) continue;
if (column >= 512) {
columnsTruncated = true;
continue;
}
cells[column] = cell;
}
if (!cells.length && !columnsTruncated) continue;
if (++rowCount > 10_000) throw new Truncation("Row limit");
for (const [column, cell] of cells.entries()) {
if (column) yield "\t";
if (!cell) continue;
const value = all(cell, "v").next().value;
const raw = value?.children.find((child) => typeof child === "string") ?? "";
if (cell.attrs.t === "s") {
const shared = strings[Number(raw)];
if (shared) yield* inCell(text(shared));
} else if (cell.attrs.t === "inlineStr") yield* inCell(text(cell));
else {
if (value && hasCellText(value)) yield* inCell(text(value));
else for (const formula of all(cell, "f")) yield* inCell(text(formula));
const formatId = formats[Number(cell.attrs.s ?? 0)]?.attrs.numFmtId;
const format = formatId ? customFormats.get(formatId) : undefined;
if (
(cell.attrs.t === undefined || cell.attrs.t === "n") &&
raw.trim() &&
formatId &&
formatId !== "0"
)
yield ` [Excel numFmtId=${formatId}${format ? `; format=${format}` : ""}; date system ${dateSystem}]`;
}
}
yield "\n";
if (columnsTruncated) throw new Truncation("Column limit");
}
}
} else if (extension === ".pptx") {
const rels = relations("ppt/_rels/presentation.xml.rels");
let index = 0;
for (const slide of all(read("ppt/presentation.xml"), "sldId")) {
yield `\n\n## Slide ${++index}\n`;
const target = rels.get(slide.attrs["r:id"] ?? slide.attrs.id);
if (!target) {
skip(`slide relationship ${index}`);
continue;
}
const name = target.startsWith("/")
? target.slice(1)
: NodePath.posix.normalize(`ppt/${target}`);
yield* paragraphs(read(name));
const slideRels = NodePath.posix.join(
NodePath.posix.dirname(name),
"_rels",
`${NodePath.posix.basename(name)}.rels`,
);
if (!files.has(slideRels)) continue;
for (const rel of all(read(slideRels), "Relationship")) {
const target = rel.attrs.Target;
if (
!rel.attrs.Type?.endsWith("/notesSlide") ||
rel.attrs.TargetMode === "External" ||
!target
)
continue;
yield "Notes:\n";
yield* paragraphs(
read(
target.startsWith("/")
? target.slice(1)
: NodePath.posix.join(NodePath.posix.dirname(name), target),
),
);
}
}
} else if (extension === ".epub") {
const rootfile = all(read("META-INF/container.xml"), "rootfile").next().value?.attrs[
"full-path"
];
if (!rootfile) {
if (!notes.length) skip("EPUB package");
yield* notes;
return;
}
const packageXml = read(rootfile);
const items = new Map(
[...all(packageXml, "item")].map((item) => [item.attrs.id, item.attrs.href]),
);
let index = 0;
for (const item of all(packageXml, "itemref")) {
yield `\n\n## Chapter ${++index}\n`;
const href = items.get(item.attrs.idref);
if (!href) {
skip(`chapter ${index}`);
continue;
}
let decoded: string;
try {
decoded = decodeURIComponent(href.split("#")[0]!);
} catch {
skip(`chapter ${index}`);
continue;
}
const name = NodePath.posix.normalize(
NodePath.posix.join(NodePath.posix.dirname(rootfile), decoded),
);
yield* paragraphs(read(name));
}
} else throw new Error("Unsupported document");
yield* notes;
}
}
export async function extractDocumentText(path: string, extension: string): Promise<string> {
let result = "";
let remaining = MAX_OUTPUT;
let cellDepth = 0;
let started = false;
let space = false;
let truncated = false;
try {
// Charge raw text before normalization so repeated whitespace also exhausts the budget.
for await (const part of document(path, extension)) {
if (part === CELL_START) {
if (cellDepth++ === 0) {
started = false;
space = false;
} else if (started) space = true;
continue;
}
if (part === CELL_END) {
cellDepth--;
if (cellDepth > 0 && started) space = true;
continue;
}
if (typeof part !== "string" && started) continue;
const raw = typeof part === "string" ? part : part.fallback;
let value = raw.slice(0, remaining).replaceAll("\0", "");
remaining -= Math.min(raw.length, remaining);
if (cellDepth) {
value = value.replaceAll(/\s+/g, " ");
const trimmed = value.trim();
if (trimmed) {
result += (started && (space || value.startsWith(" ")) ? " " : "") + trimmed;
started = true;
}
space = value.endsWith(" ") || (!value && space);
} else result += value;
if (remaining === 0) throw new Truncation("Output budget");
}
} catch (error) {
if (!(error instanceof Truncation)) throw error;
truncated = true;
}
result = result.replaceAll(/\n(?:[ \t]*\n){2,}/g, "\n\n").trim();
if (!result && !truncated) throw new Error("No text extracted");
return result + (truncated ? TRUNCATED : "");
}