sailfork / t3code-github / apps/ mobile/ src/ lib/ markdownDocument.ts
import type { Root, RootContent } from "mdast";
import { unified, type Processor } from "unified";
import remarkParse from "remark-parse";
import remarkGfm from "remark-gfm";
import { remarkKeepWindowsProsePaths } from "@t3tools/shared/markdownWindowsPaths";
import { isWindowsAbsolutePath } from "@t3tools/shared/path";
const parser = unified().use(remarkParse).use(remarkGfm).use(remarkKeepWindowsProsePaths);
export type MarkdownNode = Root | RootContent;
interface MarkdownEdit {
readonly start: number;
readonly end: number;
readonly content: string;
}
function applyMarkdownEdits(markdown: string, edits: MarkdownEdit[]): string {
if (edits.length === 0) return markdown;
const parts: string[] = [];
let offset = 0;
// AST traversal emits disjoint edits in source order.
for (const edit of edits) {
parts.push(markdown.slice(offset, edit.start), edit.content);
offset = edit.end;
}
parts.push(markdown.slice(offset));
return parts.join("");
}
function decodeCodePoint(codePoint: number, entity: string): string {
if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {
return entity;
}
return String.fromCodePoint(codePoint);
}
export function decodeHtmlEntities(value: string): string {
return value.replace(
/&(?:#(\d+)|#x([0-9a-f]+)|amp|apos|gt|lt|nbsp|quot);/gi,
(entity, decimal: string | undefined, hexadecimal: string | undefined) => {
if (decimal) {
return decodeCodePoint(Number.parseInt(decimal, 10), entity);
}
if (hexadecimal) {
return decodeCodePoint(Number.parseInt(hexadecimal, 16), entity);
}
switch (entity.toLowerCase()) {
case "&":
return "&";
case "'":
return "'";
case ">":
return ">";
case "<":
return "<";
case " ":
return "\u00a0";
case """:
return '"';
default:
return entity;
}
},
);
}
const HTML_TAG = /<!--[\s\S]*?-->|<\/?[a-z](?:[^>"']|"[^"]*"|'[^']*')*>/gi;
function attribute(tag: string, name: string): string | null {
const attributes = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g;
for (const match of tag.matchAll(attributes)) {
if (match[1]?.toLowerCase() === name) {
return decodeHtmlEntities(match[2] ?? match[3] ?? match[4] ?? "");
}
}
return null;
}
function markdownDestination(url: string): string {
return `<${url.replace(/&/g, "\\&").replace(/</g, "%3C").replace(/>/g, "%3E").replace(/\|/g, "%7C").replace(/\n/g, "")}>`;
}
const ANCHOR_TAG = /^<\/?a[\s/>]/i;
/** Replacements for the anchor tags still ahead, and how many emitted links are open. */
type AnchorState = { readonly pending: string[]; open: number };
// Code spans keep their text; an odd trailing backslash would escape the closing "]".
function escapeLinkLabel(text: string): string {
return text
.replace(/(`+)[\s\S]*?\1|\\*[[\]]/g, (match) =>
match.startsWith("`") || match.length % 2 === 0
? match
: `${match.slice(0, -1)}\\${match.slice(-1)}`,
)
.replace(/\\+$/, (slashes) => (slashes.length % 2 ? `${slashes}\\` : slashes));
}
// An unescaped "!" before the inserted "[" would turn the link into an image.
function bangBefore(text: string, index: number): boolean {
if (text[index - 1] !== "!") return false;
let backslashes = 0;
while (text[index - 2 - backslashes] === "\\") backslashes += 1;
return backslashes % 2 === 0;
}
/**
* Pairs `<a>` tags in order. Only a closed anchor with an http(s) or mailto href becomes a
* Markdown link; any other anchor keeps its text unlinked.
*/
function anchorMarkdown(tags: readonly string[]): AnchorState {
const replacements = tags.map(() => "");
const open: Array<{ index: number; href: string | undefined }> = [];
tags.forEach((tag, index) => {
if (tag[1] !== "/") {
const href = attribute(tag, "href")?.trim();
open.push({ index, href: href && /^(?:https?|mailto):/i.test(href) ? href : undefined });
return;
}
const opening = open.pop();
if (!opening?.href) return;
replacements[opening.index] = "[";
replacements[index] = `](${markdownDestination(opening.href)})`;
});
return { pending: replacements, open: 0 };
}
export function markdownHtmlTags(content: string): RegExpExecArray[] {
// An HTML block such as <details> can also contain Markdown fences and code spans.
const codeRanges: Array<{ start: number; end: number }> = [];
const tags = [...content.matchAll(HTML_TAG)];
let tagIndex = 0;
const delimiters = [...content.matchAll(/`+|~{3,}/g)].filter((delimiter) => {
while (tags[tagIndex] && tags[tagIndex]!.index + tags[tagIndex]![0].length <= delimiter.index)
tagIndex += 1;
const tag = tags[tagIndex];
return !tag || delimiter.index < tag.index;
});
for (let index = 0; index < delimiters.length; index += 1) {
const opening = delimiters[index]!;
const lineStart = content.lastIndexOf("\n", opening.index - 1) + 1;
const fenced =
opening[0].length >= 3 && /^ {0,3}$/.test(content.slice(lineStart, opening.index));
if (opening[0][0] === "~" && !fenced) continue;
let closingIndex = -1;
for (let candidate = index + 1; candidate < delimiters.length; candidate += 1) {
const closing = delimiters[candidate]!;
if (!fenced && closing[0] === opening[0]) {
closingIndex = candidate;
break;
}
if (!fenced || closing[0][0] !== opening[0][0] || closing[0].length < opening[0].length)
continue;
const start = content.lastIndexOf("\n", closing.index - 1) + 1;
const nextLine = content.indexOf("\n", closing.index);
const end = nextLine === -1 ? content.length : nextLine;
if (
/^ {0,3}$/.test(content.slice(start, closing.index)) &&
/^\s*$/.test(content.slice(closing.index + closing[0].length, end))
) {
closingIndex = candidate;
break;
}
}
if (closingIndex === -1) {
if (!fenced) continue;
// An unclosed fence runs to the end of the content, so no later opener matters.
codeRanges.push({ start: opening.index, end: content.length });
break;
}
const closing = delimiters[closingIndex]!;
codeRanges.push({ start: opening.index, end: closing.index + closing[0].length });
index = closingIndex;
}
return tags.filter(
(tag) => !codeRanges.some((range) => tag.index >= range.start && tag.index < range.end),
);
}
/**
* Called only for parsed HTML nodes, so Markdown code examples keep their literal tags.
* `anchors` is shared across sibling inline HTML nodes because each of those carries a single tag.
*/
export function markdownHtmlContent(
content: string,
options: { readonly anchors?: AnchorState; readonly inTableCell?: boolean } = {},
): string {
const tags = markdownHtmlTags(content);
const anchors =
options.anchors ??
anchorMarkdown(tags.map((tag) => tag[0]).filter((tag) => ANCHOR_TAG.test(tag)));
let result = "";
let end = 0;
for (const match of tags) {
const text = content.slice(end, match.index);
result += anchors.open > 0 ? escapeLinkLabel(text) : text;
const replacement = htmlTagMarkdown(match[0], anchors, options);
if (replacement === "[" && bangBefore(result, result.length))
result = `${result.slice(0, -1)}\\!`;
result +=
anchors.open > 0 && /[\s[]/.test(result.at(-1) ?? "")
? replacement.replace(/^ /, "")
: replacement;
end = match.index + match[0].length;
}
const text = content.slice(end);
return result + (anchors.open > 0 ? escapeLinkLabel(text) : text);
}
function htmlTagMarkdown(
tag: string,
anchors: AnchorState,
options: { readonly inTableCell?: boolean },
): string {
if (tag.startsWith("<!--")) return "";
if (ANCHOR_TAG.test(tag)) {
const replacement = anchors.pending.shift() ?? "";
if (replacement === "[") anchors.open += 1;
else if (replacement) anchors.open -= 1;
return replacement;
}
if (/^<img\b/i.test(tag)) {
const src = attribute(tag, "src");
if (!src) return "";
const alt = (attribute(tag, "alt") ?? "").replace(/[\\[\]&]/g, "\\Loading Sailfork
amp;");
return `})`;
}
// A blank line inside an open link label would end the paragraph before its "]".
const blockBreak = anchors.open === 0 ? "\n\n" : " ";
if (/^<summary\b/i.test(tag)) return `${blockBreak}**`;
if (/^<\/summary\s*>/i.test(tag)) return `**${blockBreak}`;
// A line break would end the table row, so a cell keeps its content on one line.
if (/^<br\b/i.test(tag)) return options.inTableCell ? " " : " \n";
if (/^<\/?(?:p|div|details)\b/i.test(tag)) return blockBreak;
return "";
}
export function markdownWithHtml(markdown: string): string {
if (!/<(?:!--|\/?[a-z])/i.test(markdown)) return markdown;
const edits: Array<{ start: number; end: number; content: string }> = [];
const visit = (node: MarkdownNode, inTableCell: boolean, inLink: boolean) => {
if (!("children" in node)) return;
inTableCell ||= node.type === "tableCell";
const children: readonly MarkdownNode[] = node.children;
// Inline HTML arrives one tag per node, so anchors pair across phrasing siblings.
const anchors =
node.type === "root" || node.type === "blockquote" || node.type === "listItem"
? undefined
: anchorMarkdown(
children.flatMap((child) =>
child.type === "html"
? markdownHtmlTags(child.value)
.map((tag) => tag[0])
.filter((tag) => ANCHOR_TAG.test(tag))
: [],
),
);
for (const child of children) {
const start = child.position?.start.offset;
const end = child.position?.end.offset;
const linkLabel = inLink || (anchors?.open ?? 0) > 0;
if (child.type === "text") {
if (linkLabel && start !== undefined && end !== undefined) {
const content = escapeLinkLabel(markdown.slice(start, end));
if (content !== markdown.slice(start, end)) edits.push({ start, end, content });
}
continue;
}
if (child.type !== "html") {
visit(child, inTableCell, linkLabel);
continue;
}
const content = markdownHtmlContent(child.value, { anchors, inTableCell });
if (start === undefined || end === undefined) continue;
// Text before an outermost link is never edited, so this edit cannot overlap another.
if (content.startsWith("[") && !linkLabel && bangBefore(markdown, start))
edits.push({ start: start - 1, end: start, content: "\\!" });
edits.push({ start, end, content });
}
};
visit(parser.parse(markdown), false, false);
return applyMarkdownEdits(markdown, edits);
}
// Start only at the beginning of a backtick run to avoid retrying every suffix.
const INLINE_CODE_SPAN = /(?<!`)`+([^`\r\n]+)`+/g;
export function markdownWithInlineFileLinks(markdown: string, fileLink: RegExp): string {
return transformNativeMarkdown(markdown, fileLink, false);
}
function transformNativeMarkdown(
markdown: string,
fileLink: RegExp | undefined,
tableBreaks: boolean,
): string {
const breakTables = tableBreaks && mayHaveTableUnderText(markdown);
let linkFiles = false;
if (fileLink) {
for (const [, code] of markdown.matchAll(INLINE_CODE_SPAN)) {
if (fileLink.test(code!.trim())) {
linkFiles = true;
break;
}
}
}
if (!linkFiles && !breakTables) return markdown;
const edits: MarkdownEdit[] = [];
const fileEdits: MarkdownEdit[] = [];
let rewritesPipe = false;
const visit = (node: MarkdownNode) => {
if (linkFiles && node.type === "inlineCode") {
const start = node.position?.start.offset;
const end = node.position?.end.offset;
const escaped = node.value.replace(/[\\[\]&]/g, "\\Loading Sailfork
amp;");
if (
start !== undefined &&
end !== undefined &&
fileLink?.test(node.value) &&
!/[<>]/.test(node.value)
) {
const edit = {
start,
end,
content: `[${escaped}](<${escaped}>)`,
};
edits.push(edit);
fileEdits.push(edit);
if (node.value.includes("|")) rewritesPipe = true;
}
} else if (node.type !== "link" && node.type !== "linkReference" && "children" in node)
node.children.forEach((child, index) => {
const previous = node.children[index - 1];
const start = child.position?.start;
if (
breakTables &&
child.type === "table" &&
previous?.type === "paragraph" &&
start?.offset !== undefined &&
previous.position?.end.line === start.line - 1
) {
let lineStart = start.offset;
while (lineStart > 0 && !/[\r\n]/.test(markdown[lineStart - 1]!)) lineStart -= 1;
// Replace only the container prefix, leaving cell edits disjoint from the table break.
const [, header = "", , ending = "\n", markers = ""] =
/^([ \t>]*)([^\r\n]*)(\r\n|\r|\n)([ \t>]*)/.exec(markdown.slice(lineStart)) ?? [];
edits.push({
start: lineStart,
end: lineStart + header.length,
content: `${markers.trimEnd()}${ending}${markers}`,
});
}
visit(child);
});
};
visit(parser.parse(markdown));
// Rewritten pipes can change table structure, so preserve the original pass ordering.
if (breakTables && rewritesPipe)
return markdownWithTableBreaks(applyMarkdownEdits(markdown, fileEdits));
return applyMarkdownEdits(markdown, edits);
}
// A delimiter row such as `|---|:--:|`, possibly inside a blockquote or list item.
function isTableDelimiterRow(line: string): boolean {
if (!line.includes("-")) return false;
const row = line.replace(/^[ \t>]+/, "").trim();
return row
.replace(/^\||\|$/g, "")
.split("|")
.every((cell) => /^:?-+:?$/.test(cell.trim()));
}
function mayHaveTableUnderText(markdown: string): boolean {
if (!markdown.includes("|")) return false;
const lines = markdown.split(/\r\n|\r|\n/);
for (let index = 2; index < lines.length; index += 1) {
if (isTableDelimiterRow(lines[index]!) && /[^\s>]/.test(lines[index - 2]!)) return true;
}
return false;
}
/** md4c starts tables only at paragraph starts; insert breaks to match GFM. */
export function markdownWithTableBreaks(markdown: string): string {
return transformNativeMarkdown(markdown, undefined, true);
}
// `>` covers a destination continued on a blockquote line.
const WINDOWS_DESTINATION = /\](?:\(|:)[\s>]*<?(?:[A-Za-z]:[\\/]|\\\\)/;
interface DestinationToken {
readonly start: { readonly offset: number };
readonly end: { readonly offset: number };
}
/**
* Markdown reads `C:\work\.t3` as `C:\work.t3`. Web keeps authored Windows destinations
* literally (see `remarkKeepWindowsPaths` in shared), so escape their backslashes to give
* both parsers here the same path. `\&` stays an escape, as `markdownWithHtml` emits it.
* Expects authored Markdown: already escaped output would be escaped twice.
*/
function markdownWithWindowsPathDestinations(markdown: string): string {
// Skips the extra parse for the common message that has no Windows destination.
if (!WINDOWS_DESTINATION.test(markdown)) return markdown;
const edits: MarkdownEdit[] = [];
function record(
this: { resume(): string; sliceSerialize(token: DestinationToken): string },
token: DestinationToken,
) {
this.resume();
const authored = this.sliceSerialize(token);
if (!isWindowsAbsolutePath(authored) || !authored.includes("\\") || /[<>]/.test(authored))
return;
const start = token.start.offset;
const end = token.end.offset;
const escaped = authored.replace(/\\(?!&)/g, "\\\\");
// A bare destination ends at an unescaped `)`, so wrap it to keep `\)` inside.
const angled = markdown[start - 1] === "<";
edits.push({ start, end, content: angled ? escaped : `<${escaped}>` });
}
unified()
.use(remarkParse)
.use(remarkGfm)
.use(function (this: Processor) {
(this.data().fromMarkdownExtensions ??= []).push({
exit: { resourceDestinationString: record, definitionDestinationString: record },
});
})
.parse(markdown);
return applyMarkdownEdits(markdown, edits);
}
// HTML wrappers can expose Markdown that wasn't in the first AST, so normalize them first.
export function markdownForNativeParsing(markdown: string): string {
return markdownWithWindowsPathDestinations(markdownWithHtml(markdown));
}
export function parseMarkdownDocument(markdown: string): Root {
return parser.parse(markdownForNativeParsing(markdown));
}
export function markdownForNativeRendering(markdown: string, fileLink: RegExp): string {
return transformNativeMarkdown(markdownForNativeParsing(markdown), fileLink, true);
}
/** Native media slots only cover standalone images; nested file embeds must become links. */
export function markdownWithFileEmbeds(
markdown: string,
label: (href: string, standalone: boolean) => string | null,
): { markdown: string; fileHrefs: ReadonlySet<string> } {
const fileHrefs = new Set<string>();
if (!markdown.includes("![")) return { markdown, fileHrefs };
const root = parser.parse(markdown);
const topLevelBlocks = new Set<MarkdownNode>(root.children);
const definitions = new Map<string, string>();
const collect = (node: MarkdownNode) => {
if (node.type === "definition") {
const identifier = node.identifier.toUpperCase();
if (!definitions.has(identifier)) definitions.set(identifier, node.url);
}
if ("children" in node) node.children.forEach(collect);
};
collect(root);
const edits: Array<{ start: number; end: number; content: string }> = [];
const visit = (node: MarkdownNode, standalone = false, insideLink = false) => {
if (node.type === "image" || node.type === "imageReference") {
const href =
node.type === "image" ? node.url : definitions.get(node.identifier.toUpperCase());
const name = href === undefined ? null : label(href, standalone);
const start = node.position?.start.offset;
const end = node.position?.end.offset;
if (href !== undefined && name !== null && start !== undefined && end !== undefined) {
const text = name.replace(/[\\[\]`*_~|&<>]/g, "\\Loading Sailfork
amp;");
const url = href.replace(/[\\&]/g, "\\Loading Sailfork
amp;").replace(/</g, "%3C").replace(/>/g, "%3E");
edits.push({ start, end, content: insideLink ? text : `[${text}](<${url}>)` });
fileHrefs.add(href);
}
}
if ("children" in node)
node.children.forEach((child) =>
visit(
child,
node.type === "paragraph" && node.children.length === 1 && topLevelBlocks.has(node),
insideLink || node.type === "link" || node.type === "linkReference",
),
);
};
visit(root);
const parts: string[] = [];
let offset = 0;
for (const edit of edits) {
parts.push(markdown.slice(offset, edit.start), edit.content);
offset = edit.end;
}
parts.push(markdown.slice(offset));
return { markdown: parts.join(""), fileHrefs };
}