sailfork / t3code-github / .repos/ alchemy-effect/ packages/ alchemy/ src/ Util/ Redaction.ts
import * as Redacted from "effect/Redacted";
import * as Stream from "effect/Stream";
import { isPlainObject } from "./data.ts";
/**
* Removing secrets from text that may echo them: API error bodies, Helm
* stderr, command output. Secrets come from the `Redacted` values a caller
* sent; each is matched raw, base64-encoded and JSON-escaped, since those are
* the forms an echo usually takes.
*/
/** Marker written in place of a secret unless a caller picks another. */
export const DEFAULT_REDACTION_MARKER = "<redacted>";
/**
* Plaintext strings wrapped in {@link Redacted} anywhere in `value`, including
* strings nested inside a redacted object or array. Plain values are ignored.
*/
export const collectRedactedSecrets = (value: unknown): string[] => {
const out: string[] = [];
const visit = (current: unknown, sensitive: boolean): void => {
if (Redacted.isRedacted(current)) {
visit(Redacted.value(current), true);
return;
}
if (typeof current === "number") {
if (!sensitive) return;
const text = String(current);
if (text.length === 0) return;
out.push(text);
return;
}
if (typeof current === "string") {
if (!sensitive || current.length === 0) return;
out.push(current);
// Env values are strings, so a redacted object arrives as JSON.
// Parse it so an echoed inner secret still scrubs.
const head = current.charCodeAt(0);
if (head !== 123 && head !== 91) return;
try {
const parsed: unknown = JSON.parse(current);
if (parsed !== null && typeof parsed === "object") visit(parsed, true);
} catch {
// Not JSON. The string itself is the secret.
}
return;
}
if (Array.isArray(current)) {
for (const item of current) visit(item, sensitive);
return;
}
if (isPlainObject(current)) {
for (const item of Object.values(current)) visit(item, sensitive);
}
};
visit(value, false);
return [...new Set(out)];
};
/** The forms a secret is echoed in: raw, base64, and JSON-escaped. */
const secretForms = (secret: string): string[] => [
...new Set(
[secret, Buffer.from(secret).toString("base64"), JSON.stringify(secret).slice(1, -1)].filter(
(form) => form.length > 0,
),
),
];
/**
* A marker that cannot reproduce any secret: `preferred`, unless a secret
* appears inside it, then a private-use character absent from every secret.
*/
export const redactionMarker = (
secrets: ReadonlyArray<string>,
preferred: string = DEFAULT_REDACTION_MARKER,
): string => {
if (!secrets.some((secret) => preferred.includes(secret))) return preferred;
for (let codePoint = 0xe000; codePoint <= 0xf8ff; codePoint++) {
const candidate = String.fromCodePoint(codePoint);
if (secrets.every((secret) => !secret.includes(candidate))) return candidate;
}
for (let codePoint = 0xf0000; codePoint <= 0xffffd; codePoint++) {
const candidate = String.fromCodePoint(codePoint);
if (secrets.every((secret) => !secret.includes(candidate))) return candidate;
}
// Only a secret larger than the private-use space gets here. An empty
// replacement is safer than reproducing any part of it.
return "";
};
// Secrets shorter than 4 characters only match whole tokens, so "42" does not
// eat "14293". Longer secrets match anywhere (`printf "prefix%ssuffix"`).
const SHORT_SECRET_LENGTH = 4;
const replaceForm = (text: string, form: string, anywhere: boolean, marker: string): string => {
if (form.length === 0 || !text.includes(form)) return text;
if (anywhere) return text.replaceAll(form, marker);
const pattern = form.replace(/[.*+?^${}()|[\]\\]/g, "\\Loading Sailfork
amp;");
return text.replace(new RegExp(`(?<![A-Za-z0-9_])${pattern}(?![A-Za-z0-9_])`, "g"), marker);
};
/** Replace every form of every secret in plain text with `marker`. */
export const redactText = (
text: string,
secrets: ReadonlyArray<string>,
marker: string = DEFAULT_REDACTION_MARKER,
): string => {
if (secrets.length === 0 || text.length === 0) return text;
// One secret can be a prefix of another. Replace the longest form first or
// the shorter one splits the longer one and leaves its suffix behind.
const forms = new Map<string, boolean>();
for (const secret of secrets) {
const anywhere = secret.length >= SHORT_SECRET_LENGTH;
for (const form of secretForms(secret)) {
forms.set(form, forms.get(form) === true || anywhere);
}
}
const ordered = [...forms.entries()].sort((left, right) => right[0].length - left[0].length);
let out = text;
for (const [form, anywhere] of ordered) out = replaceForm(out, form, anywhere, marker);
return out;
};
const redactNode = (value: unknown, secrets: ReadonlyArray<string>, marker: string): unknown => {
if (typeof value === "string") return redactText(value, secrets, marker);
if (typeof value === "number" && secrets.some((secret) => secret === String(value))) {
return marker;
}
if (Array.isArray(value)) return value.map((item) => redactNode(item, secrets, marker));
if (isPlainObject(value)) {
return Object.fromEntries(
Object.entries(value).map(([key, child]) => [key, redactNode(child, secrets, marker)]),
);
}
return value;
};
/**
* Like {@link redactText}, but a JSON document is redacted leaf by leaf, so a
* secret that only appears JSON-escaped inside a string is still caught.
* Text that is not JSON falls back to {@link redactText}.
*/
export const redactJson = (
text: string,
secrets: ReadonlyArray<string>,
marker: string = DEFAULT_REDACTION_MARKER,
): string => {
if (secrets.length === 0 || text.length === 0) return text;
try {
return JSON.stringify(redactNode(JSON.parse(text), secrets, marker));
} catch {
return redactText(text, secrets, marker);
}
};
/**
* Redact a text stream without leaking a secret split across chunks. Every
* form of every secret matches as a substring; the earliest match wins, and
* the longest form wins a tie. A match is only replaced once no form starting
* at or before it can still grow in the next chunk, so the output does not
* depend on where the chunks split.
*/
export const redactStream = <E, R>(
stream: Stream.Stream<string, E, R>,
secrets: ReadonlyArray<string>,
marker: string = DEFAULT_REDACTION_MARKER,
): Stream.Stream<string, E, R> => {
const forms = [...new Set(secrets.flatMap(secretForms))].sort(
(left, right) => right.length - left.length,
);
if (forms.length === 0) return stream;
// Split `text` into redacted output and the pending suffix still held back.
// At the end of the stream nothing can grow, so nothing is held back.
const scan = (text: string, final: boolean): readonly [pending: string, output: string] => {
let remaining = text;
let output = "";
while (remaining.length > 0) {
let matchIndex = -1;
let match: string | undefined;
for (const form of forms) {
const index = remaining.indexOf(form);
if (
index >= 0 &&
(matchIndex < 0 ||
index < matchIndex ||
(index === matchIndex && form.length > (match?.length ?? 0)))
) {
matchIndex = index;
match = form;
}
}
// The longest suffix that is a proper prefix of some form.
let suffixLength = 0;
if (!final) {
for (const form of forms) {
const candidateLength = Math.min(form.length - 1, remaining.length);
for (let length = candidateLength; length > suffixLength; length--) {
if (form.startsWith(remaining.slice(-length))) {
suffixLength = length;
break;
}
}
}
}
const pendingIndex = remaining.length - suffixLength;
// A form that may still complete at or before the match would win
// over it, so wait for more text before deciding.
if (match !== undefined && (suffixLength === 0 || matchIndex < pendingIndex)) {
output += remaining.slice(0, matchIndex) + marker;
remaining = remaining.slice(matchIndex + match.length);
continue;
}
output += remaining.slice(0, pendingIndex);
remaining = remaining.slice(pendingIndex);
break;
}
return [remaining, output] as const;
};
return stream.pipe(
Stream.mapAccum(
() => "",
(pending, chunk) => {
const [remaining, output] = scan(pending + chunk, false);
return [remaining, output.length === 0 ? [] : [output]] as const;
},
{
onHalt: (pending) => {
const [, output] = scan(pending, true);
return output.length === 0 ? [] : [output];
},
},
),
);
};