import { use, type CSSProperties } from "react";
import { resolveDiffThemeName } from "../../lib/diffRendering";
import type { EmbeddedScript } from "../../lib/embeddedScripts";
import { getSyntaxHighlighterPromise } from "../../lib/syntaxHighlighting";
interface SyntaxToken {
readonly content: string;
readonly offset: number;
readonly color?: string;
readonly fontStyle?: number;
}
type TokenLines = ReadonlyArray<ReadonlyArray<SyntaxToken>>;
const NO_EMBEDDED_SCRIPTS: ReadonlyArray<EmbeddedScript> = [];
function syntaxTokenStyle(token: SyntaxToken): CSSProperties {
const fontStyle = token.fontStyle ?? 0;
return {
...(token.color ? { color: token.color } : {}),
...(fontStyle & 1 ? { fontStyle: "italic" } : {}),
...(fontStyle & 2 ? { fontWeight: 700 } : {}),
...(fontStyle & 4 ? { textDecoration: "underline" } : {}),
};
}
/** Suspends during grammar loading; use plain text as the Suspense fallback. */
export function HighlightedTokens({
code,
language,
embedded = NO_EMBEDDED_SCRIPTS,
theme,
wordClassName,
}: {
code: string;
language: string;
embedded?: ReadonlyArray<EmbeddedScript>;
theme: "light" | "dark";
wordClassName?: string;
}) {
const highlighter = use(getSyntaxHighlighterPromise(language));
for (const script of embedded) use(getSyntaxHighlighterPromise(script.language));
const themeName = resolveDiffThemeName(theme);
const tokenize = (text: string, lang: string) =>
highlighter.codeToTokens(text, { lang, theme: themeName }).tokens;
const lines = keyedLines(
code,
withEmbeddedScripts(code, tokenize(code, language), embedded, tokenize),
);
// The compiler requires JSX to recognize this component; `use` alone does not suffice.
return (
<>
{lines.map(({ key, tokens, ending }) => (
<span key={key}>
{wordClassName
? wordsOf(tokens).map((part) =>
typeof part === "string" ? (
part
) : (
<span key={part.key} className={wordClassName}>
{part.pieces.map((piece) => (
<span key={piece.key} style={syntaxTokenStyle(piece.token)}>
{piece.text}
</span>
))}
</span>
),
)
: tokens.map((token) => (
<span key={`${token.offset}:${token.content}`} style={syntaxTokenStyle(token)}>
{token.content}
</span>
))}
{ending}
</span>
))}
</>
);
}
export function withEmbeddedScripts(
code: string,
lines: TokenLines,
embedded: ReadonlyArray<EmbeddedScript>,
tokenize: (text: string, language: string) => TokenLines,
): TokenLines {
if (embedded.length === 0) return lines;
const styles = Array.from<SyntaxToken | undefined>({ length: code.length });
for (const token of lines.flat()) {
styles.fill(token, token.offset, token.offset + token.content.length);
}
for (const script of embedded) {
let scriptLines: TokenLines;
try {
scriptLines = tokenize(script.text, script.language);
} catch {
continue;
}
for (const token of scriptLines.flat()) {
if (token.content === "") continue;
const last = token.offset + token.content.length - 1;
styles.fill(token, script.starts[token.offset], script.ends[last]);
}
}
return restyledLines(code, styles);
}
function restyledLines(
code: string,
styles: ReadonlyArray<SyntaxToken | undefined>,
): SyntaxToken[][] {
const lines: SyntaxToken[][] = [];
let start = 0;
for (const lineBreak of [...code.matchAll(/\r?\n/gu), undefined]) {
const end = lineBreak?.index ?? code.length;
const tokens: SyntaxToken[] = [];
for (let index = start; index < end;) {
const style = styles[index];
let next = index + 1;
while (
next < end &&
styles[next]?.color === style?.color &&
styles[next]?.fontStyle === style?.fontStyle
) {
next += 1;
}
tokens.push({
content: code.slice(index, next),
offset: index,
...(style?.color ? { color: style.color } : {}),
...(style?.fontStyle ? { fontStyle: style.fontStyle } : {}),
});
index = next;
}
lines.push(tokens);
if (lineBreak) start = lineBreak.index + lineBreak[0].length;
}
return lines;
}
/** Restore CRLF from source; Shiki strips both line-ending forms. */
export function keyedLines<Token extends SyntaxToken>(
code: string,
lines: ReadonlyArray<ReadonlyArray<Token>>,
) {
const endings = code.match(/\r?\n/gu) ?? [];
let lineStart = 0;
return lines.map((tokens, index) => {
const text = tokens.map((token) => token.content).join("");
const key = `${lineStart}:${text}`;
const ending = endings[index] ?? "";
lineStart += text.length + ending.length;
return { key, tokens, ending };
});
}
interface Word {
readonly key: string;
readonly pieces: Array<{ key: string; text: string; token: SyntaxToken }>;
}
export function wordsOf(tokens: ReadonlyArray<SyntaxToken>): Array<string | Word> {
const parts: Array<string | Word> = [];
let word: Word | null = null;
for (const token of tokens) {
let offset = token.offset;
for (const text of token.content.split(/(\s+)/u)) {
if (text === "") continue;
if (/^\s/u.test(text)) {
parts.push(text);
word = null;
} else {
if (!word) {
word = { key: String(offset), pieces: [] };
parts.push(word);
}
word.pieces.push({ key: String(offset), text, token });
}
offset += text.length;
}
}
return parts;
}