// Generates the provider API reference under website/src/content/docs/providers
// (plus the providers sidebar and reference redirects) from source JSDoc.
//
// Usage: node scripts/generate-api-reference.ts [--copy-markers]
// --copy-markers (or API_REFERENCE_COPY_MARKERS=1) emits the website's
// dev-time copy-editor markers.
import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
import * as NodeServices from "@effect/platform-node/NodeServices";
import { Command, Flag } from "effect/cli";
import * as Config from "effect/Config";
import * as Console from "effect/Console";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import type * as PlatformError from "effect/PlatformError";
import * as ts from "tsrs/unstable/ast";
import type { Node, SourceFile } from "tsrs/unstable/ast";
import {
docCommentLines,
jsdocCopyId,
LINK_TAG_RE,
markDocLines,
normalizeLinkTarget,
parseLinkTag,
} from "./jsdoc-blocks.ts";
import { createSyntaxProject } from "./typescript-source.ts";
interface SourceRoot {
/** Directory scanned for documented source files (repo-relative). */
srcRoot: string;
/**
* Synthetic provider name for flat single-provider packages (their files
* sit directly at `srcRoot`); empty for the alchemy package, whose
* top-level directories ARE the providers.
*/
providerPrefix: string;
/** Display prefix for the page's `Source:` line. */
sourceDisplayPrefix: string;
}
/** Source roots, relative to the repo root (resolved in `main`). */
const sourceRoots: SourceRoot[] = [
{ srcRoot: "packages/alchemy/src", providerPrefix: "", sourceDisplayPrefix: "src" },
{
srcRoot: "packages/better-auth/src",
providerPrefix: "BetterAuth",
sourceDisplayPrefix: "packages/better-auth/src",
},
{ srcRoot: "packages/git/src", providerPrefix: "Git", sourceDisplayPrefix: "packages/git/src" },
{
srcRoot: "packages/drizzle/src",
providerPrefix: "Drizzle",
sourceDisplayPrefix: "packages/drizzle/src",
},
{
srcRoot: "packages/acme/src",
providerPrefix: "ACME",
sourceDisplayPrefix: "packages/acme/src",
},
{
srcRoot: "packages/axiom/src",
providerPrefix: "Axiom",
sourceDisplayPrefix: "packages/axiom/src",
},
{
srcRoot: "packages/doppler/src",
providerPrefix: "Doppler",
sourceDisplayPrefix: "packages/doppler/src",
},
{
srcRoot: "packages/infisical/src",
providerPrefix: "Infisical",
sourceDisplayPrefix: "packages/infisical/src",
},
{
srcRoot: "packages/turso/src",
providerPrefix: "Turso",
sourceDisplayPrefix: "packages/turso/src",
},
{
srcRoot: "packages/planetscale/src",
providerPrefix: "Planetscale",
sourceDisplayPrefix: "packages/planetscale/src",
},
{
srcRoot: "packages/neon/src",
providerPrefix: "Neon",
sourceDisplayPrefix: "packages/neon/src",
},
{
srcRoot: "packages/prisma/src",
providerPrefix: "Prisma",
sourceDisplayPrefix: "packages/prisma/src",
},
{
srcRoot: "packages/hetzner/src",
providerPrefix: "Hetzner",
sourceDisplayPrefix: "packages/hetzner/src",
},
{
srcRoot: "packages/railway/src",
providerPrefix: "Railway",
sourceDisplayPrefix: "packages/railway/src",
},
{ srcRoot: "packages/fly/src", providerPrefix: "Fly", sourceDisplayPrefix: "packages/fly/src" },
{
srcRoot: "packages/github/src",
providerPrefix: "GitHub",
sourceDisplayPrefix: "packages/github/src",
},
{
srcRoot: "packages/stripe/src",
providerPrefix: "Stripe",
sourceDisplayPrefix: "packages/stripe/src",
},
{
srcRoot: "packages/cloudflare/src",
providerPrefix: "Cloudflare",
sourceDisplayPrefix: "packages/cloudflare/src",
},
{ srcRoot: "packages/gcp/src", providerPrefix: "GCP", sourceDisplayPrefix: "packages/gcp/src" },
{ srcRoot: "packages/aws/src", providerPrefix: "AWS", sourceDisplayPrefix: "packages/aws/src" },
{
srcRoot: "packages/domain/src",
providerPrefix: "Domain",
sourceDisplayPrefix: "packages/domain/src",
},
{
srcRoot: "packages/kubernetes/src",
providerPrefix: "Kubernetes",
sourceDisplayPrefix: "packages/kubernetes/src",
},
{
srcRoot: "packages/docker/src",
providerPrefix: "Docker",
sourceDisplayPrefix: "packages/docker/src",
},
{ srcRoot: "packages/sql/src", providerPrefix: "SQL", sourceDisplayPrefix: "packages/sql/src" },
];
interface FileEntry {
/** Output-relative path (provider-prefixed for flat packages). */
relativePath: string;
absolutePath: string;
/** Display path for the page's `Source:` blockquote. */
sourceDisplay: string;
}
interface ExampleBlock {
title: string;
body: string;
}
interface ExampleSection {
title: string;
description: string;
examples: ExampleBlock[];
}
interface PageDoc {
title: string;
/** Display path for the `Source:` blockquote. */
sourceDisplay: string;
summary: string;
sections: ExampleSection[];
/** True for `@layer` pages — renders the Layer metadata line. */
isLayer: boolean;
/** Service tags the Layer provides (`@provides`). */
provides: string[];
/** Optional peer dependencies (`@peer`). */
peers: string[];
}
// Paths are POSIX on every platform we run on; also fold Windows separators.
const normalizeSlashes = (value: string) => value.replaceAll("\\", "/");
const isSourceFile = (baseName: string) =>
(baseName.endsWith(".ts") || baseName.endsWith(".tsx")) &&
!baseName.endsWith(".d.ts") &&
baseName !== "index.ts";
const discoverFiles = Effect.fn(function* (repoRoot: string, root: SourceRoot) {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const srcRoot = path.join(repoRoot, root.srcRoot);
const entries: FileEntry[] = [];
// Flat single-provider packages: every file under srcRoot belongs to the
// synthetic provider directory.
if (root.providerPrefix) {
const files = yield* fs.readDirectory(srcRoot, { recursive: true });
for (const file of files) {
if (!isSourceFile(path.basename(file))) continue;
entries.push({
relativePath: path.join(root.providerPrefix, file),
absolutePath: path.join(srcRoot, file),
sourceDisplay: `${root.sourceDisplayPrefix}/${normalizeSlashes(file)}`,
});
}
entries.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
return entries;
}
for (const dir of yield* fs.readDirectory(srcRoot)) {
const dirPath = path.join(srcRoot, dir);
const info = yield* fs.stat(dirPath);
if (info.type !== "Directory") continue;
const files = yield* fs
.readDirectory(dirPath, { recursive: true })
.pipe(Effect.orElseSucceed((): string[] => []));
for (const file of files) {
if (!isSourceFile(path.basename(file))) continue;
const relativePath = path.join(dir, file);
entries.push({
relativePath,
absolutePath: path.join(srcRoot, relativePath),
sourceDisplay: `${root.sourceDisplayPrefix}/${normalizeSlashes(relativePath)}`,
});
}
}
entries.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
return entries;
});
function getJsDocText(node: Node): string {
return node.jsDoc?.map((doc) => doc.getText()).join("\n") ?? "";
}
function cleanDocComment(raw: string): string {
return raw
.replace(/^\/\*\*?/, "")
.replace(/\*\/$/, "")
.split("\n")
.map((line) => line.replace(/^\s*\*\s?/, ""))
.join("\n");
}
interface ParsedJSDoc {
summary: string;
sections: ExampleSection[];
hasResourceTag: boolean;
hasBindingTag: boolean;
/** `@layer` — a Layer factory providing a Context service. */
hasLayerTag: boolean;
/**
* `@primitive` — a building block exported as a module's `make` (e.g.
* `Domain.Event`). Its page is named after the file, not the export.
*/
hasPrimitiveTag: boolean;
/** `@provides <Service.Tag>` — service tags the Layer satisfies. */
provides: string[];
/** `@peer <package>` — optional peer dependencies the Layer needs. */
peers: string[];
category: string;
product: string;
/** `@title <Heading>` — overrides the section heading (the export name). */
title: string;
}
/** Where a declaration's JSDoc lives, for copy markers. */
interface MarkerContext {
/** Repo-relative path of the source file. */
file: string;
/** Full source text (JSDoc offsets index into it). */
text: string;
}
function docLines(node: Node, raw: string, markers?: MarkerContext): string[] {
const docs = node.jsDoc ?? [];
// Markers need one comment per declaration to map blocks to source.
const start = docs[0]?.getStart();
// `getStart()` can land on leading `//` trivia; only map clean comments.
if (
markers &&
docs.length === 1 &&
start !== undefined &&
markers.text.startsWith("/**", start)
) {
return markDocLines(docCommentLines(markers.text, start), (block) =>
jsdocCopyId(markers.file, start, block),
);
}
return cleanDocComment(raw).split("\n");
}
function parseJSDoc(node: Node, markers?: MarkerContext): ParsedJSDoc {
const raw = getJsDocText(node);
if (!raw) {
return {
summary: "",
sections: [],
hasResourceTag: false,
hasBindingTag: false,
hasLayerTag: false,
hasPrimitiveTag: false,
provides: [],
peers: [],
category: "",
product: "",
title: "",
};
}
const lines = docLines(node, raw, markers);
const summaryLines: string[] = [];
const sections: ExampleSection[] = [];
const provides: string[] = [];
const peers: string[] = [];
let hasResourceTag = false;
let hasBindingTag = false;
let hasLayerTag = false;
let hasPrimitiveTag = false;
let category = "";
let product = "";
let title = "";
let sawTag = false;
let currentSection: ExampleSection | undefined;
let currentExample: ExampleBlock | undefined;
let sectionDescLines: string[] = [];
let collectingSectionDesc = false;
let insideFence = false;
const flushExample = () => {
if (!currentExample) return;
currentExample.body = currentExample.body.trim();
if (!currentSection) {
currentSection = { title: "Examples", description: "", examples: [] };
sections.push(currentSection);
}
currentSection.examples.push(currentExample);
currentExample = undefined;
};
const flushSectionDesc = () => {
if (currentSection && sectionDescLines.length > 0) {
currentSection.description = sectionDescLines.join("\n").trim();
}
sectionDescLines = [];
collectingSectionDesc = false;
};
for (const line of lines) {
// Track fenced code blocks so an `@`-prefixed line inside an example
// (e.g. a decorator) is never mistaken for a JSDoc tag.
if (line.trim().startsWith("```")) {
insideFence = !insideFence;
}
const proseHeading = insideFence
? null
: line.trim().match(/^###\s+(.+?)\s+<!-- api-prose -->$/);
if (proseHeading) {
if (!sawTag) summaryLines.push(`### ${proseHeading[1]!.trim()}`);
continue;
}
const section = insideFence ? null : line.trim().match(/^###\s+(.+)$/);
if (section) {
sawTag = true;
flushExample();
flushSectionDesc();
currentSection = { title: section[1]!.trim(), description: "", examples: [] };
sections.push(currentSection);
collectingSectionDesc = true;
continue;
}
const example = insideFence ? null : line.trim().match(/^\*\*Example:\*\*\s*(.*)$/);
if (example) {
sawTag = true;
flushSectionDesc();
flushExample();
currentExample = { title: example[1]!.trim() || "Example", body: "" };
continue;
}
const tag = insideFence ? null : line.trim().match(/^@(\w+)\s*(.*)$/);
if (tag) {
sawTag = true;
const [, name, rest] = tag;
const value = (rest ?? "").trim();
switch (name) {
case "resource":
hasResourceTag = true;
break;
case "binding":
hasBindingTag = true;
break;
case "layer":
hasLayerTag = true;
break;
case "primitive":
hasPrimitiveTag = true;
break;
case "provides":
if (value) provides.push(value);
break;
case "peer":
if (value) peers.push(value);
break;
case "category":
case "group":
if (value) category = value;
break;
case "product":
case "label":
if (value) product = value;
break;
case "title":
if (value) title = value;
break;
case "section":
flushExample();
flushSectionDesc();
currentSection = { title: value || "Examples", description: "", examples: [] };
sections.push(currentSection);
collectingSectionDesc = true;
break;
case "example":
flushSectionDesc();
flushExample();
currentExample = { title: value || "Example", body: "" };
break;
}
continue;
}
if (!sawTag) {
summaryLines.push(line);
continue;
}
if (currentExample) {
currentExample.body += `${line}\n`;
} else if (collectingSectionDesc) {
sectionDescLines.push(line);
}
}
flushSectionDesc();
flushExample();
return {
summary: summaryLines.join("\n").trim(),
sections,
hasResourceTag,
hasBindingTag,
hasLayerTag,
hasPrimitiveTag,
provides,
peers,
category,
product,
title,
};
}
function declName(node: Node): string {
if (ts.isVariableStatement(node)) {
return node.declarationList.declarations[0]?.name.getText() ?? "";
}
if (
ts.isClassDeclaration(node) ||
ts.isFunctionDeclaration(node) ||
ts.isInterfaceDeclaration(node) ||
ts.isTypeAliasDeclaration(node)
) {
return node.name?.getText() ?? "";
}
return "";
}
interface Primary {
name: string;
doc: ParsedJSDoc;
category: string;
product: string;
}
const hasContent = (doc: ParsedJSDoc) => Boolean(doc.summary) || doc.sections.length > 0;
/**
* Map a public export name back to its local declaration name when a file
* re-exports under an alias, e.g. `export { VpcLinkResource as VpcLink }`
* lets us find the documented `VpcLinkResource` const from the tagged
* `VpcLink` interface.
*/
export function exportedNames(sourceFile: SourceFile): string[] {
const names = new Set<string>();
const addBinding = (name: ts.BindingName): void => {
if (ts.isIdentifier(name)) names.add(name.text);
else
for (const element of name.elements) {
if (ts.isBindingElement(element) && element.name) addBinding(element.name);
}
};
for (const statement of sourceFile.statements) {
if (ts.isExportDeclaration(statement)) {
if (statement.exportClause && ts.isNamedExports(statement.exportClause)) {
for (const spec of statement.exportClause.elements) names.add(spec.name.text);
} else if (statement.exportClause && ts.isNamespaceExport(statement.exportClause)) {
names.add(statement.exportClause.name.text);
}
} else if (ts.isExportAssignment(statement)) {
if (!statement.isExportEquals) names.add("default");
} else if (
(ts.isVariableStatement(statement) ||
ts.isClassDeclaration(statement) ||
ts.isFunctionDeclaration(statement) ||
ts.isInterfaceDeclaration(statement) ||
ts.isTypeAliasDeclaration(statement) ||
ts.isEnumDeclaration(statement) ||
ts.isModuleDeclaration(statement) ||
ts.isImportEqualsDeclaration(statement)) &&
statement.modifierFlags & ts.ModifierFlags.Export
) {
if (statement.modifierFlags & ts.ModifierFlags.Default) names.add("default");
else if (ts.isVariableStatement(statement)) {
for (const declaration of statement.declarationList.declarations)
addBinding(declaration.name);
} else if (
ts.isClassDeclaration(statement) ||
ts.isFunctionDeclaration(statement) ||
ts.isInterfaceDeclaration(statement) ||
ts.isTypeAliasDeclaration(statement) ||
ts.isEnumDeclaration(statement) ||
ts.isModuleDeclaration(statement) ||
ts.isImportEqualsDeclaration(statement)
) {
if (statement.name) names.add(statement.name.text);
}
}
}
return [...names];
}
function localNameForExport(sourceFile: SourceFile, publicName: string): string | undefined {
for (const ed of sourceFile.statements.filter(ts.isExportDeclaration)) {
if (ed.moduleSpecifier) continue;
for (const spec of ed.exportClause && ts.isNamedExports(ed.exportClause)
? ed.exportClause.elements
: []) {
if (spec.name.text === publicName) {
return (spec.propertyName ?? spec.name).text;
}
}
}
return undefined;
}
/**
* The page for a file is owned by the single exported declaration tagged
* `@resource` or `@binding`, and named after it. Authors sometimes write the
* docs (summary/@section/@example) on a sibling declaration of the same name
* (an `interface X` paired with `const X`) or on an internal const that is
* re-exported under the tagged name (the ApiGateway `XResource as X` pattern).
* When the tagged declaration itself has no content, pull it from that related
* declaration so the page isn't dropped as empty.
*/
export function findTaggedPrimary(
sourceFile: SourceFile,
markers?: MarkerContext,
): Primary | undefined {
const candidates: Node[] = [
...sourceFile.statements
.filter(ts.isVariableStatement)
.filter((s) => Boolean(s.modifierFlags & ts.ModifierFlags.Export)),
...sourceFile.statements
.filter(ts.isClassDeclaration)
.filter((c) => Boolean(c.modifierFlags & ts.ModifierFlags.Export)),
...sourceFile.statements
.filter(ts.isInterfaceDeclaration)
.filter((i) => Boolean(i.modifierFlags & ts.ModifierFlags.Export)),
// An overloaded function carries its JSDoc on the first signature.
...sourceFile.statements
.filter(ts.isFunctionDeclaration)
.filter((f) => Boolean(f.modifierFlags & ts.ModifierFlags.Export)),
];
for (const node of candidates) {
const doc = parseJSDoc(node, markers);
if (!doc.hasResourceTag && !doc.hasBindingTag && !doc.hasLayerTag && !doc.hasPrimitiveTag) {
continue;
}
// A primitive is its module's `make`; the page is named after the module.
const exportName = doc.hasPrimitiveTag
? normalizeSlashes(sourceFile.fileName)
.split("/")
.pop()!
.replace(/\.tsx?$/, "")
: declName(node);
if (!exportName) continue;
const name = doc.title || exportName;
const category = doc.category;
const product = doc.product;
if (hasContent(doc)) return { name, doc, category, product };
// Tagged declaration has no prose — look for the related declaration that
// carries the docs (same name, or re-exported under this name).
const localName = localNameForExport(sourceFile, exportName);
const related: Node[] = [
...sourceFile.statements.filter(ts.isVariableStatement),
...sourceFile.statements.filter(ts.isClassDeclaration),
...sourceFile.statements.filter(ts.isInterfaceDeclaration),
...sourceFile.statements.filter(ts.isTypeAliasDeclaration),
].filter((d) => {
if (d === node) return false;
const dn = declName(d);
return dn === exportName || (localName !== undefined && dn === localName);
});
let best: ParsedJSDoc | undefined;
for (const d of related) {
const pd = parseJSDoc(d, markers);
if (pd.sections.length > 0) {
best = pd;
break;
}
if (!best && pd.summary) best = pd;
}
// Borrowed prose keeps the TAGGED declaration's kind + metadata.
const merged: ParsedJSDoc = {
...(best ?? doc),
hasResourceTag: doc.hasResourceTag,
hasBindingTag: doc.hasBindingTag,
hasLayerTag: doc.hasLayerTag,
hasPrimitiveTag: doc.hasPrimitiveTag,
provides: doc.provides,
peers: doc.peers,
};
return { name, doc: merged, category, product };
}
return undefined;
}
/** Resolves a `{@link}` symbol target to a generated page URL, if any. */
type LinkResolver = (target: string) => string | undefined;
/**
* Build a per-page resolver factory. A symbol resolves to another generated
* page preferring (in order): a same-directory page named after it, a
* same-directory page whose source file exports it (an access-level service
* like `ReadNamespace` documented on its `ReadWriteNamespace` page), a unique
* name match within the same provider, a unique exporter within the same
* provider, then a unique global name match. Member suffixes
* (`Cluster#capacityProviders`), provider-qualified names (`Cloudflare.Worker`)
* and `FooProps` interfaces resolve to the base resource's page.
*/
function makeLinkResolverFactory(pages: PageEntry[]): (fromDir: string) => LinkResolver {
const byName = new Map<string, PageEntry[]>();
const byExport = new Map<string, PageEntry[]>();
for (const p of pages) {
if (!byName.has(p.resource)) byName.set(p.resource, []);
byName.get(p.resource)!.push(p);
for (const name of p.exports) {
if (!byExport.has(name)) byExport.set(name, []);
byExport.get(name)!.push(p);
}
}
const providers = new Set(pages.map((p) => p.provider));
return (fromDir: string) => {
const fromProvider = fromDir.split("/")[0] ?? "";
const lookup = (name: string, provider?: string): PageEntry | undefined => {
const scope = (list: PageEntry[] | undefined) =>
(list ?? []).filter((c) => !provider || c.provider === provider);
const named = scope(byName.get(name));
const exporting = scope(byExport.get(name));
const sameDirNamed = named.filter((c) => c.dir === fromDir);
if (sameDirNamed.length === 1) return sameDirNamed[0];
const sameDirExporting = exporting.filter((c) => c.dir === fromDir);
if (sameDirExporting.length === 1) return sameDirExporting[0];
const providerNamed = named.filter((c) => c.provider === fromProvider);
if (providerNamed.length === 1) return providerNamed[0];
const providerExporting = exporting.filter((c) => c.provider === fromProvider);
if (providerExporting.length === 1) return providerExporting[0];
if (named.length === 1) return named[0];
return undefined;
};
return (target: string) => {
// Strip a `#member` suffix — the link lands on the owning resource page.
const base = target.split("#")[0];
const attempts: { name: string; provider?: string }[] = [{ name: base }];
if (base.includes(".")) {
const segments = base.split(".");
const first = segments[0];
const last = segments[segments.length - 1];
if (providers.has(first)) {
// Provider-qualified: `Cloudflare.Worker`, `Cloudflare.KV.Namespace`.
attempts.push({ name: last, provider: first });
} else {
// Member access on a symbol: `KeyPairProps.publicKeyMaterial`.
attempts.push({ name: first });
if (first.endsWith("Props")) {
attempts.push({ name: first.slice(0, -"Props".length) });
}
}
} else if (base.endsWith("Props")) {
attempts.push({ name: base.slice(0, -"Props".length) });
}
for (const attempt of attempts) {
const found = lookup(attempt.name, attempt.provider);
if (found) return found.link;
}
return undefined;
};
};
}
const isUrl = (target: string) => /^https?:\/\//.test(target);
/**
* Replace `{@link ...}` tags with markdown links, skipping fenced code
* blocks. Symbols that don't resolve to a generated page render as inline
* code (or their label) instead of leaking the raw tag.
*/
/** Symbol targets that didn't resolve to a page, for the end-of-run summary. */
const unresolvedLinkTargets = new Map<string, number>();
function linkifyMarkdown(markdown: string, resolve: LinkResolver): string {
const replaceTags = (chunk: string) =>
chunk.replace(LINK_TAG_RE, (_, inner: string) => {
// A tag wrapped across source lines carries the line break in its
// label — collapse it so the markdown link stays intact.
const parsed = parseLinkTag(inner.replace(/\s+/g, " "));
const label = parsed.label;
if (isUrl(parsed.target)) {
return `[${label ?? parsed.target}](${parsed.target})`;
}
const target = normalizeLinkTarget(parsed.target);
const url = resolve(target);
if (!url) {
unresolvedLinkTargets.set(target, (unresolvedLinkTargets.get(target) ?? 0) + 1);
}
const text = label ?? `\`${target}\``;
return url ? `[${text}](${url})` : text;
});
// Apply across contiguous non-fence chunks (a `{@link}` may span lines);
// leave fenced code blocks untouched.
const out: string[] = [];
let chunk: string[] = [];
let insideFence = false;
const flushChunk = () => {
if (chunk.length > 0) {
out.push(replaceTags(chunk.join("\n")));
chunk = [];
}
};
for (const line of markdown.split("\n")) {
if (line.trim().startsWith("```")) {
if (!insideFence) flushChunk();
insideFence = !insideFence;
out.push(line);
continue;
}
if (insideFence) {
out.push(line);
} else {
chunk.push(line);
}
}
flushChunk();
return out.join("\n");
}
function yamlString(value: string): string {
if (/[\n:"{}[\],&*?|>!%@`#]/.test(value) || value.trim() !== value) {
return JSON.stringify(value);
}
return value;
}
function renderPageBody(doc: PageDoc): string {
const parts: string[] = [];
if (doc.summary) {
parts.push(doc.summary);
}
for (const section of doc.sections) {
const secParts = [`## ${section.title}`];
if (section.description) {
secParts.push(section.description);
}
for (const example of section.examples) {
if (section.examples.length > 1) {
secParts.push(`**${example.title}**`);
}
secParts.push(example.body);
}
parts.push(secParts.join("\n\n"));
}
return parts.join("\n\n");
}
function renderReferenceFrontmatter(title: string): string {
return [
"---",
`title: ${yamlString(`${title} reference`)}`,
`description: ${yamlString(`Resources and capabilities for ${title}.`)}`,
// Keep the existing search-engine indexing policy during this experiment.
"head:",
" - tag: meta",
" attrs:",
" name: robots",
' content: "noindex, follow"',
"prev: false",
"next: false",
"tableOfContents:",
" minHeadingLevel: 2",
" maxHeadingLevel: 2",
"---",
].join("\n");
}
function renderResource(doc: PageDoc, resolve: LinkResolver): string {
const headerLines = [`> **Source:** \`${doc.sourceDisplay}\``];
if (doc.isLayer) {
const meta = ["**Kind:** Layer"];
if (doc.provides.length > 0) {
meta.push(`**Provides:** ${doc.provides.map((tag) => `\`${tag}\``).join(", ")}`);
}
if (doc.peers.length > 0) {
meta.push(`**Peer dependencies:** ${doc.peers.map((peer) => `\`${peer}\``).join(", ")}`);
}
headerLines.push(`> ${meta.join(" · ")}`);
}
const sourceBlock = headerLines.join("\n");
const body = linkifyMarkdown(renderPageBody(doc).trim(), resolve);
if (body) {
return `${sourceBlock}\n\n${body}\n`;
}
return `${sourceBlock}\n`;
}
/** Change this grouping to experiment with larger or smaller reference pages. */
function referenceLocation(outputRelative: string, product: string) {
const parts = normalizeSlashes(outputRelative).replace(/\.md$/, "").split("/");
// Flat providers declare service-sized pages with @product instead of
// folders; a resource without one gets its own page.
const group =
parts.length > 2
? parts.slice(0, 2)
: product
? [
parts[0],
"reference",
product
.toLowerCase()
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-|-$/g, ""),
]
: parts;
const title = parts.slice(parts.length > 2 ? 2 : 1).join("-");
return {
outputRelative: `${group.join("/")}.md`,
title:
parts.length > 2 ? group.join(".") : product ? `${parts[0]}.${product}` : parts.join("."),
resourceTitle: title,
link: `/providers/${group.join("/").toLowerCase()}#${title.toLowerCase()}`,
};
}
/** Keep example headings below their resource, with resource-scoped slugs. */
function nestResourceHeadings(markdown: string, resource: string): string {
let fence: string | undefined;
return markdown
.split("\n")
.map((line) => {
const marker = line.match(/^\s{0,3}(`{3,}|~{3,})/);
if (marker) {
if (!fence) fence = marker[1];
else if (marker[1][0] === fence[0] && marker[1].length >= fence.length) fence = undefined;
return line;
}
if (fence) return line;
return line.replace(
/^(#{1,6})\s+(.+)$/,
(_, hashes: string, title: string) =>
`${"#".repeat(Math.min(6, Math.max(3, hashes.length + 1)))} ${resource}: ${title}`,
);
})
.join("\n");
}
/** Providers shown first in the sidebar; the rest follow alphabetically. */
const PROVIDER_ORDER = ["AWS", "Cloudflare"];
interface SidebarLeaf {
label: string;
link: string;
}
interface SidebarGroup {
label: string;
collapsed: true;
items: SidebarItem[];
}
type SidebarItem = SidebarLeaf | SidebarGroup;
interface PageEntry {
provider: string;
service: string;
resource: string;
category: string;
product: string;
link: string;
/** Source directory relative to srcRoot (slash-normalized), e.g. `Cloudflare/KV`. */
dir: string;
/** All names the page's source file exports — `{@link}` targets documented on this page. */
exports: string[];
}
const byLabel = (a: { label: string }, b: { label: string }) => a.label.localeCompare(b.label);
function orderedKeys(keys: string[], order: string[]): string[] {
const ranked = keys.filter((k) => order.includes(k));
ranked.sort((a, b) => order.indexOf(a) - order.indexOf(b));
const rest = keys.filter((k) => !order.includes(k)).sort();
return [...ranked, ...rest];
}
/** The sidebar lists documents; resource anchors live in the page's TOC. */
function buildServiceItems(pages: PageEntry[]): SidebarItem[] {
return pages
.map((page) => ({ label: page.product || page.service || page.resource, link: page.link }))
.sort(byLabel);
}
function buildProvidersSidebar(entries: PageEntry[]): SidebarItem[] {
const byProvider = new Map<string, PageEntry[]>();
for (const e of entries) {
if (!byProvider.has(e.provider)) byProvider.set(e.provider, []);
byProvider.get(e.provider)!.push(e);
}
const providers: SidebarGroup[] = [];
for (const provider of orderedKeys([...byProvider.keys()], PROVIDER_ORDER)) {
const byPage = new Map<string, PageEntry[]>();
for (const entry of byProvider.get(provider)!) {
const link = entry.link.split("#")[0];
if (!byPage.has(link)) byPage.set(link, []);
byPage.get(link)!.push(entry);
}
const pages = [...byPage].map(([link, resources]) => ({
...resources[0],
link,
product: resources.every((resource) => resource.product === resources[0].product)
? resources[0].product
: "",
}));
const items = buildServiceItems(pages);
providers.push({ label: provider, collapsed: true, items });
}
assertNoDuplicateSiblings(providers, []);
return providers;
}
/**
* Duplicate sibling labels are always a tagging bug (e.g. two products
* resolving to the same name in one category) and render as confusing
* twin sections — fail the generation instead of shipping them.
*/
function assertNoDuplicateSiblings(items: SidebarItem[], path: string[]) {
const seen = new Map<string, number>();
for (const item of items) {
seen.set(item.label, (seen.get(item.label) ?? 0) + 1);
}
const dups = [...seen.entries()].filter(([, n]) => n > 1);
if (dups.length > 0) {
throw new Error(
`Duplicate sidebar sibling label(s) under "${path.join(" > ") || "(root)"}": ${dups
.map(([label, n]) => `"${label}" ×${n}`)
.join(", ")} — fix the @product/@category tags on the offending files.`,
);
}
for (const item of items) {
if ("items" in item) {
assertNoDuplicateSiblings(item.items, [...path, item.label]);
}
}
}
/**
* Writes `content` only when it differs from what's on disk, so a dev server
* watching the output reloads just the pages that changed.
*/
const writeIfChanged = Effect.fn(function* (file: string, content: string) {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const current = yield* fs
.readFileString(file)
.pipe(Effect.orElseSucceed((): string | undefined => undefined));
if (current === content) return false;
yield* fs.makeDirectory(path.dirname(file), { recursive: true });
yield* fs.writeFileString(file, content);
return true;
});
/** Deletes files under `dir` not in `keep`, then any directories left empty. */
const pruneOutput = (
dir: string,
keep: Set<string>,
): Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem | Path.Path> =>
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const entries = yield* fs.readDirectory(dir).pipe(Effect.orElseSucceed((): string[] => []));
for (const name of entries) {
const full = path.join(dir, name);
const info = yield* fs.stat(full);
if (info.type === "Directory") {
yield* pruneOutput(full, keep);
const left = yield* fs.readDirectory(full).pipe(Effect.orElseSucceed((): string[] => []));
if (left.length === 0) yield* fs.remove(full, { recursive: true }).pipe(Effect.ignore);
} else if (!keep.has(full)) {
yield* fs.remove(full, { force: true });
}
}
});
class DuplicateReferenceAnchor extends Data.TaggedError("DuplicateReferenceAnchor")<{
readonly message: string;
}> {}
const command = Command.make(
"generate-api-reference",
{
copyMarkers: Flag.Boolean("copy-markers").pipe(
Flag.withDefault(false),
Flag.withDescription(
"Emit copy-editor markers for the website's dev server (also API_REFERENCE_COPY_MARKERS=1)",
),
),
},
Effect.fn(function* (flags) {
const path = yield* Path.Path;
const fs = yield* FileSystem.FileSystem;
const repoRoot = path.resolve(import.meta.dirname, "..");
const websiteRoot = path.join(repoRoot, "website");
const envCopyMarkers = yield* Config.String("API_REFERENCE_COPY_MARKERS").pipe(
Config.withDefault(""),
);
const config = {
outRoot: path.join(websiteRoot, "src/content/docs/providers"),
/**
* Emit `<!--copy:jsdoc:...-->` markers after each prose block so the
* website's dev-time copy editor can map rendered text back to the JSDoc
* it came from. Dev only — production builds never set this.
*/
copyMarkers: envCopyMarkers === "1" || flags.copyMarkers,
};
yield* fs.makeDirectory(config.outRoot, { recursive: true });
const outputs = new Set<string>();
const seen = new Map<string, string>();
const pageEntries: PageEntry[] = [];
const pending: { outputRelative: string; product: string; doc: PageDoc }[] = [];
let written = 0;
const redirects: Record<string, string> = {};
const anchors = new Set<string>();
let skipped = 0;
for (const root of sourceRoots) {
const entries = yield* discoverFiles(repoRoot, root);
yield* Console.log(`Discovered ${entries.length} source files in ${root.srcRoot}.`);
yield* Effect.scoped(
Effect.gen(function* () {
const syntax = yield* createSyntaxProject(entries.map((entry) => entry.absolutePath));
for (const entry of entries) {
const sourceFile = yield* syntax.getSourceFile(entry.absolutePath);
const primary = findTaggedPrimary(
sourceFile,
config.copyMarkers
? {
file: normalizeSlashes(path.relative(repoRoot, entry.absolutePath)),
text: sourceFile.text,
}
: undefined,
);
if (!primary) {
skipped++;
continue;
}
// Only emit a page when there's actual documented content; a bare
// frontmatter + source link stub is noise.
if (!primary.doc.summary && primary.doc.sections.length === 0) {
skipped++;
continue;
}
// Mirror the source directory structure; name the page after the
// tagged declaration (e.g. Cloudflare.AI.Search/AiSearchInstance.md).
const relDir = path.dirname(entry.relativePath);
const outputRelative = path.join(relDir, `${primary.name}.md`);
const existing = seen.get(outputRelative);
if (existing) {
yield* Console.warn(
` collision: ${outputRelative} from ${entry.relativePath} (already from ${existing})`,
);
}
seen.set(outputRelative, entry.relativePath);
const doc: PageDoc = {
title: primary.name,
sourceDisplay: entry.sourceDisplay,
summary: primary.doc.summary,
sections: primary.doc.sections,
isLayer: primary.doc.hasLayerTag,
provides: primary.doc.provides,
peers: primary.doc.peers,
};
pending.push({ outputRelative, product: primary.product, doc });
const exportNames = exportedNames(sourceFile);
const segments = normalizeSlashes(outputRelative).split("/");
const location = referenceLocation(outputRelative, primary.product);
const oldLink = `/providers/${normalizeSlashes(outputRelative).replace(/\.md$/, "").toLowerCase()}`;
if (anchors.has(location.link)) {
return yield* new DuplicateReferenceAnchor({
message: `Duplicate reference anchor: ${location.link}`,
});
}
anchors.add(location.link);
// A resource whose page kept its old URL needs no redirect.
if (oldLink !== location.link.split("#")[0]) redirects[oldLink] = location.link;
pageEntries.push({
provider: segments[0] ?? "",
service: segments.length > 2 ? segments[1] : "",
resource: primary.name,
category: primary.category,
product: primary.product,
link: location.link,
dir: normalizeSlashes(relDir),
exports: exportNames,
});
}
}),
);
}
// Second pass: render with `{@link}` resolution — the full page set must be
// known before symbol targets can resolve to their pages.
const resolverFor = makeLinkResolverFactory(pageEntries);
const groups = new Map<string, { title: string; sections: string[] }>();
for (const page of pending) {
const location = referenceLocation(page.outputRelative, page.product);
const resolve = resolverFor(normalizeSlashes(path.dirname(page.outputRelative)));
const group = groups.get(location.outputRelative) ?? { title: location.title, sections: [] };
group.sections.push(
`## ${location.resourceTitle}\n\n${nestResourceHeadings(renderResource(page.doc, resolve), location.resourceTitle)}`,
);
groups.set(location.outputRelative, group);
}
for (const [relative, group] of groups) {
const outputPath = path.join(config.outRoot, relative);
outputs.add(outputPath);
if (
yield* writeIfChanged(
outputPath,
`${renderReferenceFrontmatter(group.title)}\n\n${group.sections.join("\n\n")}\n`,
)
) {
written++;
}
}
const sidebar = buildProvidersSidebar(pageEntries);
// Landing page for the Reference tab (/providers): a provider directory.
// Each provider's reference belongs to its docs hub — the directory just
// routes there (or into the tree, for providers without a hub). The
// ProviderDirectory component reads the generated sidebar for counts.
// Regenerated with the rest of the tree on every run.
const referenceIndex = [
"---",
"title: API Reference",
"description: Every provider alchemy can manage — pick a provider to open its docs hub and resource reference.",
"---",
"",
'import ProviderDirectory from "../../../components/ProviderDirectory.astro";',
"",
"Every resource alchemy can manage, documented from the source JSDoc and",
"organized by provider. A provider's reference lives in its docs hub —",
"pick one below, or search with `⌘K`.",
"",
"<ProviderDirectory />",
"",
].join("\n");
const indexPath = path.join(config.outRoot, "index.mdx");
outputs.add(indexPath);
yield* writeIfChanged(indexPath, referenceIndex);
yield* pruneOutput(config.outRoot, outputs);
const sidebarPath = path.join(websiteRoot, "src/generated/providers-sidebar.json");
yield* writeIfChanged(
path.join(websiteRoot, "src/generated/reference-redirects.json"),
`${JSON.stringify(redirects, null, 2)}\n`,
);
// astro.config imports the sidebar, so rewriting it restarts a dev server.
yield* writeIfChanged(sidebarPath, `${JSON.stringify(sidebar, null, 2)}\n`);
yield* Console.log(
`Done. Wrote ${written} changed reference pages containing ${pending.length} resources (skipped ${skipped} untagged${
config.copyMarkers ? ", with copy markers" : ""
}) to ${normalizeSlashes(path.relative(repoRoot, config.outRoot))}.`,
);
if (unresolvedLinkTargets.size > 0) {
const list = [...unresolvedLinkTargets.entries()]
.sort((a, b) => b[1] - a[1])
.map(([target, n]) => (n > 1 ? `${target} ×${n}` : target))
.join(", ");
yield* Console.log(`{@link} targets without a page (rendered as inline code): ${list}`);
}
yield* Console.log(
`Wrote provider sidebar to ${normalizeSlashes(path.relative(repoRoot, sidebarPath))}.`,
);
}),
).pipe(Command.withDescription("Generate the provider API reference from source JSDoc"));
Command.run(command, { version: "0.0.0" }).pipe(
Effect.provide(NodeServices.layer),
NodeRuntime.runMain,
);