sailfork / t3code-github / apps/ server/ src/ speech/ customWords.ts
import type { SpeechCustomWords } from "@t3tools/contracts";
/** Fuzzy-matches preferred terms and rewrites aliases to their preferred spelling. */
export function makeSpeechTextCorrector(
words: readonly string[],
dictionary: SpeechCustomWords = [],
) {
const MATCH_THRESHOLD = 0.18;
const matchKey = (value: string) =>
[...value]
.filter((character) => /[\p{L}\p{N}]/u.test(character))
.join("")
.toLocaleLowerCase();
const isAsciiKey = (value: string) => /^[a-z0-9]+$/i.test(value);
const soundex = (value: string) => {
const upper = value.toUpperCase();
const first = upper[0] ?? "";
const code = (character: string) => {
if ("BFPV".includes(character)) return "1";
if ("CGJKQSXZ".includes(character)) return "2";
if ("DT".includes(character)) return "3";
if (character === "L") return "4";
if ("MN".includes(character)) return "5";
if (character === "R") return "6";
return "0";
};
let previous = code(first);
let result = first;
for (const character of upper.slice(1)) {
const next = code(character);
if (next !== "0" && next !== previous) result += next;
previous = next;
}
return `${result}000`.slice(0, 4);
};
const editDistance = (left: string, right: string) => {
let previous = Array.from({ length: right.length + 1 }, (_, index) => index);
for (let leftIndex = 0; leftIndex < left.length; leftIndex += 1) {
const current = [leftIndex + 1];
for (let rightIndex = 0; rightIndex < right.length; rightIndex += 1) {
current.push(
Math.min(
current[rightIndex]! + 1,
previous[rightIndex + 1]! + 1,
previous[rightIndex]! + (left[leftIndex] === right[rightIndex] ? 0 : 1),
),
);
}
previous = current;
}
return previous[right.length]!;
};
const preserveCase = (original: string, replacement: string) => {
const letters = [...original].filter((character) => /\p{L}/u.test(character));
if (letters.length > 0 && letters.every((character) => character === character.toUpperCase()))
return replacement.toUpperCase();
const firstLetter = [...original].find((character) => /\p{L}/u.test(character));
if (firstLetter && firstLetter === firstLetter.toUpperCase()) {
const [first = "", ...rest] = [...replacement];
return first.toUpperCase() + rest.join("");
}
return replacement;
};
type CustomWordKey = { readonly word: string; readonly key: string };
const customWordKeys = (words: readonly string[]): CustomWordKey[] =>
words.flatMap((word) => {
const keys = [matchKey(word)];
if (word.includes("&")) keys.push(matchKey(word.replaceAll("&", " and ")));
return [...new Set(keys)].filter(isAsciiKey).map((key) => ({ word, key }));
});
const possessive = /['’]s$/iu;
const innerPossessives = (value: string) => value.match(/['’]s(?=\s)/giu)?.length ?? 0;
type Match = { readonly word: string; readonly score: number; readonly suffix: string };
const bestMatch = (phrase: string, keys: readonly CustomWordKey[]) => {
const key = matchKey(phrase);
const suffix = phrase.match(possessive)?.[0] ?? "";
const stem = suffix ? key.slice(0, -1) : key;
if (!isAsciiKey(stem) || stem.length > 50) return undefined;
const inner = innerPossessives(phrase);
let best: Match | undefined;
for (const custom of keys) {
if (inner > 0 && inner > innerPossessives(custom.word)) continue;
const keepSuffix = suffix !== "" && !possessive.test(custom.word);
const candidate = keepSuffix ? stem : key;
if (Math.min(candidate.length, custom.key.length) < 4 && candidate !== custom.key) continue;
const longest = Math.max(candidate.length, custom.key.length);
if (Math.abs(candidate.length - custom.key.length) > Math.max(longest * 0.25, 2)) continue;
const distance = editDistance(candidate, custom.key) / longest;
const phonetic = /^[a-z]+$/i.test(candidate) && /^[a-z]+$/i.test(custom.key);
const score =
phonetic && soundex(candidate) === soundex(custom.key) ? distance * 0.3 : distance;
if (score < MATCH_THRESHOLD && (!best || score < best.score))
best = { word: custom.word, score, suffix: keepSuffix ? suffix : "" };
}
return best;
};
const apply = (text: string): string => {
if (keys.length === 0) return text;
// Leave surrounding punctuation in the source; only horizontal whitespace can join words.
const tokens = [...text.matchAll(/[\p{L}\p{M}\p{N}](?:\S*[\p{L}\p{M}\p{N}])?/gu)]
.filter(([word]) => /^[\p{L}\p{M}\p{N}'’-]+$/u.test(word))
.map((match) => ({
value: match[0],
start: match.index,
end: match.index + match[0].length,
}));
const output: string[] = [];
let cursor = 0;
for (let index = 0; index < tokens.length;) {
let best: (Match & { readonly count: number }) | undefined;
for (let count = Math.min(4, tokens.length - index); count >= 1; count -= 1) {
const slice = tokens.slice(index, index + count);
if (
slice
.slice(0, -1)
.some(
(token, offset) =>
!/^[\t\p{Zs}]+$/u.test(text.slice(token.end, slice[offset + 1]!.start)),
)
)
continue;
const match = bestMatch(text.slice(slice[0]!.start, slice[count - 1]!.end), keys);
if (
match &&
(!best || match.score < best.score || (match.score === best.score && count < best.count))
)
best = { count, ...match };
}
if (!best) {
index += 1;
continue;
}
const first = tokens[index]!;
output.push(
text.slice(cursor, first.start),
preserveCase(first.value, best.word),
best.suffix,
);
cursor = tokens[index + best.count - 1]!.end;
index += best.count;
}
output.push(text.slice(cursor));
return output.join("");
};
const keys = customWordKeys(words);
const escapePattern = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\Loading Sailfork
amp;");
const spellings = dictionary
.flatMap(({ term, aliases }) => [term, ...aliases].map((spelling) => ({ spelling, term })))
.sort((left, right) => right.spelling.length - left.spelling.length);
if (spellings.length === 0) return apply;
const lookup = new Map(
spellings.map(({ spelling, term }) => [spelling.toLocaleLowerCase(), term]),
);
const word = "[\\p{L}\\p{M}\\p{N}\\p{Pc}]";
const pattern = new RegExp(
`(?<!${word})(?:${spellings.map(({ spelling }) => escapePattern(spelling)).join("|")})(?!${word})`,
"giu",
);
return (text: string) => {
let output = "";
let cursor = 0;
for (const match of text.matchAll(pattern)) {
output += apply(text.slice(cursor, match.index));
output += lookup.get(match[0].toLocaleLowerCase()) ?? match[0];
cursor = match.index + match[0].length;
}
return output + apply(text.slice(cursor));
};
}
export function applySpeechAliases(text: string, words: SpeechCustomWords): string {
return makeSpeechTextCorrector([], words)(text);
}
const normalizeWord = (word: string) => word.replace(/\s+/g, " ").trim();
export function normalizeSpeechCustomWords(words: SpeechCustomWords): SpeechCustomWords {
const seen = new Set<string>();
return words
.flatMap(({ term, aliases }) => {
const preferred = normalizeWord(term);
const key = preferred.toLocaleLowerCase();
if (!preferred || preferred.length > 50 || seen.has(key)) return [];
seen.add(key);
const normalizedAliases = aliases.flatMap((alias) => {
const value = normalizeWord(alias);
const aliasKey = value.toLocaleLowerCase();
if (!value || value.length > 50 || seen.has(aliasKey)) return [];
seen.add(aliasKey);
return [value];
});
return [{ term: preferred, aliases: normalizedAliases.slice(0, 8) }];
})
.slice(0, 100);
}