sailfork / t3code-github / packages/ shared/ src/ composerInlineTokens.ts

import { markdownMathRanges } from "./markdownMath.ts";

export type ComposerInlineToken =
  | {
      readonly type: "mention";
      readonly value: string;
      readonly source: string;
      readonly start: number;
      readonly end: number;
    }
  | {
      readonly type: "skill";
      readonly value: string;
      readonly source: string;
      readonly start: number;
      readonly end: number;
    };

export interface CollectComposerInlineTokensOptions {
  readonly preserveTrailingFrom?: ReadonlyArray<ComposerInlineToken>;
}

const SKILL_MENTION_SOURCE =
  /(^|\s)\p{Sc}(?![0-9][0-9_]*(?:[kKmMbBtT]|[eE][0-9]+)?(?:\s|$))(?=[a-zA-Z0-9:_-]*[a-zA-Z])([a-zA-Z0-9][a-zA-Z0-9:_-]*)/u
    .source;
const SKILL_TOKEN_REGEX = new RegExp(`${SKILL_MENTION_SOURCE}(?=\\s)`, "gu");
export const SKILL_MENTION_PATTERN = new RegExp(`${SKILL_MENTION_SOURCE}(?=\\s|$)`, "gu");
const MENTION_TOKEN_REGEX = /(^|\s)@(?:"((?:\\.|[^"\\])*)"|([^\s@"]+))(?=\s)/g;
// Bound unmatched-bracket rescans to avoid quadratic work; only basenames survive, and common filesystems cap them at 255.
const MAX_FILE_LINK_LABEL_LENGTH = 512;
const FILE_LINK_TOKEN_REGEX = new RegExp(
  `(^|\\s)\\[((?:\\\\.|[^\\]\\\\]){0,${MAX_FILE_LINK_LABEL_LENGTH}})\\]\\(([^)\\s]+)\\)(?=\\s)`,
  "g",
);
const URI_SCHEME_REGEX = /^[A-Za-z][A-Za-z0-9+.-]*:/;
const WINDOWS_DRIVE_PATH_REGEX = /^[A-Za-z]:[\\/]/;
const SCOPED_PACKAGE_REFERENCE_REGEX =
  /^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*(?:\/[^\s@"]+)*$/;
const DECORATOR_CALL_REFERENCE_REGEX = /^[\p{ID_Start}_$][\p{ID_Continue}.$\u200c\u200d]*\(/u;

function collectMentionTokens(text: string): ComposerInlineToken[] {
  const matches: ComposerInlineToken[] = [];

  for (const match of text.matchAll(FILE_LINK_TOKEN_REGEX)) {
    const fullMatch = match[0];
    const prefix = match[1] ?? "";
    const label = (match[2] ?? "").replace(/\\(.)/g, "$1");
    const encodedPath = match[3] ?? "";
    let path = encodedPath;
    try {
      path = decodeURIComponent(encodedPath);
    } catch {}
    const separatorIndex = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\"));
    const basename = separatorIndex >= 0 ? path.slice(separatorIndex + 1) : path;
    const hasExternalScheme = URI_SCHEME_REGEX.test(path) && !WINDOWS_DRIVE_PATH_REGEX.test(path);
    if (!path || hasExternalScheme || label !== basename) {
      continue;
    }
    const start = (match.index ?? 0) + prefix.length;
    const end = start + fullMatch.length - prefix.length;
    matches.push({
      type: "mention",
      value: path,
      source: text.slice(start, end),
      start,
      end,
    });
  }

  for (const match of text.matchAll(MENTION_TOKEN_REGEX)) {
    const fullMatch = match[0];
    const prefix = match[1] ?? "";
    const quotedPath = match[2];
    const path = quotedPath !== undefined ? quotedPath.replace(/\\(.)/g, "$1") : (match[3] ?? "");
    if (
      !path ||
      (quotedPath === undefined &&
        (SCOPED_PACKAGE_REFERENCE_REGEX.test(path) || DECORATOR_CALL_REFERENCE_REGEX.test(path)))
    ) {
      continue;
    }
    const start = (match.index ?? 0) + prefix.length;
    const end = start + fullMatch.length - prefix.length;
    matches.push({
      type: "mention",
      value: path,
      source: text.slice(start, end),
      start,
      end,
    });
  }

  return matches;
}

/**
 * Formulas in composer text, ignoring `

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	packages/shared/src/composerInlineTokens.ts at main · sailfork/t3code-github · Sailfork
	
		
and `\[` inside file links such as `posts.$postId.tsx`. * Links are masked with letters, not spaces, so a leading link cannot become an indented code * block. Bare `@path` mentions stay unmasked so `@README.md packages/shared/src/composerInlineTokens.ts at main · sailfork/t3code-github · Sailfork
can still close a formula. */ export function composerMathRanges( text: string, mentions: ReadonlyArray<ComposerInlineToken> = collectMentionTokens(text), ) { let masked = text; for (const mention of mentions) { if (mention.type !== "mention" || !mention.source.startsWith("[")) continue; masked = masked.slice(0, mention.start) + "x".repeat(mention.end - mention.start) + masked.slice(mention.end); } return markdownMathRanges(masked).map(({ start, end }) => ({ start, end, source: text.slice(start, end), })); } export function collectComposerInlineTokens( text: string, options: CollectComposerInlineTokensOptions = {}, ): ReadonlyArray<ComposerInlineToken> { const matches = collectMentionTokens(text); for (const match of text.matchAll(SKILL_TOKEN_REGEX)) { const fullMatch = match[0]; const prefix = match[1] ?? ""; const value = match[2] ?? ""; if (!value) { continue; } const start = (match.index ?? 0) + prefix.length; const end = start + fullMatch.length - prefix.length; matches.push({ type: "skill", value, source: text.slice(start, end), start, end, }); } for (const token of options.preserveTrailingFrom ?? []) { if ( token.end === text.length && text.slice(token.start, token.end) === token.source && !matches.some( (match) => match.type === token.type && match.start === token.start && match.end === token.end, ) ) { matches.push(token); } } // TeX can contain literal skill and file tokens, especially inside \text{}. // They remain formula source rather than becoming composer context chips. // Confirmed trailing file links are masked too, so a kept chip is not read as TeX. const math = matches.length > 0 ? composerMathRanges(text, matches) : []; return matches .filter( (match) => !math.some((formula) => match.start < formula.end && match.end > formula.start), ) .sort((left, right) => left.start - right.start); }
packages/shared/src/composerInlineTokens.ts at main · sailfork/t3code-github · Sailfork
and `\\[` inside file links such as `posts.$postId.tsx`.\n * Links are masked with letters, not spaces, so a leading link cannot become an indented code\n * block. Bare `@path` mentions stay unmasked so `@README.md packages/shared/src/composerInlineTokens.ts at main · sailfork/t3code-github · Sailfork
can still close a formula.\n */\nexport function composerMathRanges(\n text: string,\n mentions: ReadonlyArray\u003cComposerInlineToken> = collectMentionTokens(text),\n) {\n let masked = text;\n for (const mention of mentions) {\n if (mention.type !== \"mention\" || !mention.source.startsWith(\"[\")) continue;\n masked =\n masked.slice(0, mention.start) +\n \"x\".repeat(mention.end - mention.start) +\n masked.slice(mention.end);\n }\n return markdownMathRanges(masked).map(({ start, end }) => ({\n start,\n end,\n source: text.slice(start, end),\n }));\n}\n\nexport function collectComposerInlineTokens(\n text: string,\n options: CollectComposerInlineTokensOptions = {},\n): ReadonlyArray\u003cComposerInlineToken> {\n const matches = collectMentionTokens(text);\n\n for (const match of text.matchAll(SKILL_TOKEN_REGEX)) {\n const fullMatch = match[0];\n const prefix = match[1] ?? \"\";\n const value = match[2] ?? \"\";\n if (!value) {\n continue;\n }\n const start = (match.index ?? 0) + prefix.length;\n const end = start + fullMatch.length - prefix.length;\n matches.push({\n type: \"skill\",\n value,\n source: text.slice(start, end),\n start,\n end,\n });\n }\n\n for (const token of options.preserveTrailingFrom ?? []) {\n if (\n token.end === text.length &&\n text.slice(token.start, token.end) === token.source &&\n !matches.some(\n (match) =>\n match.type === token.type && match.start === token.start && match.end === token.end,\n )\n ) {\n matches.push(token);\n }\n }\n\n // TeX can contain literal skill and file tokens, especially inside \\text{}.\n // They remain formula source rather than becoming composer context chips.\n // Confirmed trailing file links are masked too, so a kept chip is not read as TeX.\n const math = matches.length > 0 ? composerMathRanges(text, matches) : [];\n return matches\n .filter(\n (match) => !math.some((formula) => match.start \u003c formula.end && match.end > formula.start),\n )\n .sort((left, right) => left.start - right.start);\n}\n","preview":null,"history":null,"version":"992a9d08e477489408844c796c13f718a75064a8"},"/_session?shell=1":null}