sailfork / t3code-github / .repos/ alchemy-effect/ website/ scripts/ generate-llms-txt.ts

/**
 * Generates `public/llms.txt` and `public/llms-full.txt` — navigation
 * indexes of the docs pages, grouped by section, with title + description
 * pulled from each page's frontmatter.
 *
 * `llms.txt` covers the guides and concept docs and links to
 * `llms-full.txt`, which additionally lists every generated service-level
 * API reference page (the Providers section) and is much larger.
 *
 * Run with: `bun scripts/generate-llms-txt.ts`
 *
 * Section ordering, headings, and prose intros are configured here.
 * Page metadata (title, description) comes from the source frontmatter,
 * so editing a page's frontmatter is enough to update llms.txt.
 */

import { readdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";

const here = path.dirname(fileURLToPath(import.meta.url));
const docsDir = path.resolve(here, "../src/content/docs");
const outFile = path.resolve(here, "../public/llms.txt");
const outFullFile = path.resolve(here, "../public/llms-full.txt");
const siteUrl = "https://alchemy.run";

interface Page {
  /** URL path, e.g. "/infrastructure-as-effects/binding" */
  href: string;
  /** Path relative to docs dir without extension, e.g. "infrastructure-as-effects/binding" */
  slug: string;
  title: string;
  description: string;
  draft: boolean;
  /** `sidebar.order` from frontmatter; `Infinity` when unset. */
  order: number;
}

interface Section {
  /** H2 heading */
  heading: string;
  /** Optional prose paragraph after the heading. */
  intro?: string;
  /**
   * Pages to include. Either a list of explicit slugs (relative to docs dir,
   * no extension) in the desired order, or a directory to enumerate
   * alphabetically.
   */
  pages: { slugs: string[] } | { directory: string; exclude?: string[] };
}

const SECTIONS: Section[] = [
  {
    heading: "Start here",
    pages: {
      slugs: ["what-is-alchemy", "getting-started", "migrating-from-v1"],
    },
  },
  {
    heading: "CLI",
    intro:
      "Every command runs through your package manager: bun alchemy <command>. Command pages by role, the Programmatic API for running them from scripts, plus the Adopting Resources and Inspecting State guides.",
    pages: {
      slugs: [
        "cli/index",
        "cli/api",
        "cli/deploy",
        "cli/plan",
        "cli/destroy",
        "cli/drift",
        "cli/nuke",
        "cli/adopting-resources",
        "cli/dev",
        "cli/logs",
        "cli/profile",
        "cli/state",
        "cli/inspecting-state",
        "cli/aws",
        "cli/cloudflare",
      ],
    },
  },
  {
    heading: "Infrastructure as Code",
    intro:
      "The noun graph and its semantics: Stacks of Resources and Actions, data flow via Inputs & Outputs and References, deploy behavior via the Resource lifecycle and Providers.",
    pages: {
      slugs: [
        "infrastructure-as-code/stack",
        "infrastructure-as-code/resource",
        "infrastructure-as-code/action",
        "infrastructure-as-code/outputs",
        "infrastructure-as-code/references",
        "infrastructure-as-code/resource-lifecycle",
        "infrastructure-as-code/renaming",
        "infrastructure-as-code/provider",
        "infrastructure-as-code/custom-provider",
        "infrastructure-as-code/local-provider",
        "infrastructure-as-code/dns-adapters",
        "infrastructure-as-code/contributions",
      ],
    },
  },
  {
    heading: "Infrastructure as Effects",
    intro:
      "How app code and infrastructure compose: Runtime, Bindings, the Construction and Runtime Phases, and Layers.",
    pages: {
      slugs: [
        "infrastructure-as-effects/runtime",
        "infrastructure-as-effects/binding",
        "infrastructure-as-effects/layers",
        "infrastructure-as-effects/event-sources",
        "infrastructure-as-effects/sinks",
        "infrastructure-as-effects/phases",
        "infrastructure-as-effects/circular-bindings",
        "infrastructure-as-effects/custom-runtime",
        "infrastructure-as-effects/telemetry",
      ],
    },
  },
  {
    heading: "APIs",
    intro:
      "Typed calls between Functions and Servers: schemaless RPC for internal communication, Effect RPC and Effect HTTP for trust boundaries.",
    pages: {
      slugs: ["apis/index", "apis/schemaless", "apis/effect-rpc", "apis/effect-http"],
    },
  },
  {
    heading: "Environments — the same app in many places",
    intro:
      "Stages, credential profiles, auth providers, secrets and config, local development, and CI.",
    pages: {
      slugs: [
        "environments/stages",
        "environments/profiles",
        "environments/auth-providers",
        "environments/custom-auth-provider",
        "environments/secrets",
        "environments/secret-providers",
        "environments/doppler",
        "environments/infisical",
        "environments/local-development",
        "environments/worktrees",
        "environments/ci",
      ],
    },
  },
  {
    heading: "State Store",
    pages: { slugs: ["state-store/index", "state-store/custom-state-store"] },
  },
  {
    heading: "Project structure",
    intro:
      "Scaling the codebase: file conventions for one stack, monorepos for many — single Stack or one per package.",
    pages: {
      slugs: [
        "project-structure/file-layout",
        "project-structure/monorepo",
        "project-structure/monorepo-single-stack",
        "project-structure/monorepo-multi-stack",
      ],
    },
  },
  {
    heading: "Testing & observability",
    intro:
      "Tests run against real clouds: the model, the end-to-end walkthrough, provider-lifecycle testing, and the harness reference.",
    pages: {
      slugs: [
        "testing/index",
        "testing/testing-a-stack",
        "testing/testing-providers",
        "testing/test-harness",
        "testing/observability",
      ],
    },
  },
  {
    heading: "Cloudflare — start here",
    intro:
      "The Cloudflare hub: overview (resources + recipes), setup (install, account, OAuth vs API token, profiles), and local development.",
    pages: { slugs: ["cloudflare/index", "cloudflare/setup", "cloudflare/local-development"] },
  },
  {
    heading: "Cloudflare — tutorial",
    intro:
      "A linear six-part walkthrough from zero to a tested, locally-developed, CI-deployed, observable Cloudflare project. Each part builds on the previous one.",
    pages: { directory: "cloudflare/tutorial" },
  },
  {
    heading: "Cloudflare — Compute",
    pages: { directory: "cloudflare/compute" },
  },
  {
    heading: "Cloudflare — Frontend",
    pages: { directory: "cloudflare/frontend" },
  },
  {
    heading: "Cloudflare — APIs",
    pages: { directory: "cloudflare/apis" },
  },
  {
    heading: "Cloudflare — Data",
    pages: { directory: "cloudflare/data" },
  },
  {
    heading: "Cloudflare — Messaging & events",
    pages: { directory: "cloudflare/messaging" },
  },
  {
    heading: "Cloudflare — Email",
    pages: { directory: "cloudflare/email" },
  },
  {
    heading: "Cloudflare — AI",
    pages: { directory: "cloudflare/ai" },
  },
  {
    heading: "Cloudflare — Security & secrets",
    pages: { directory: "cloudflare/security" },
  },
  {
    heading: "Cloudflare — Observability",
    pages: { directory: "cloudflare/observability" },
  },
  {
    heading: "Cloudflare — Networking",
    pages: { directory: "cloudflare/networking" },
  },
  {
    heading: "Prisma — start here",
    pages: { slugs: ["prisma/index", "prisma/setup"] },
  },
  {
    heading: "Prisma — tutorial",
    intro:
      "A four-part walkthrough: a Project, an Effect-native Compute API, Postgres, and a Vite frontend.",
    pages: { directory: "prisma/tutorial" },
  },
  {
    heading: "Prisma — Compute",
    pages: { directory: "prisma/compute" },
  },
  {
    heading: "Prisma — Data",
    pages: { directory: "prisma/data" },
  },
  {
    heading: "Prisma — Frontend",
    pages: { directory: "prisma/frontend" },
  },
  {
    heading: "Prisma — Guides",
    pages: { directory: "prisma/guides" },
  },
  {
    heading: "AWS — start here",
    intro:
      "The AWS hub: overview (runtimes + resources + recipes), setup (credentials, profiles, region), local development, and the Lambda vs ECS vs EKS vs EC2 decision page.",
    pages: {
      slugs: ["aws/index", "aws/setup", "aws/local-development", "aws/compute/choosing-a-runtime"],
    },
  },
  {
    heading: "AWS — tutorial",
    intro:
      "A linear five-part walkthrough from zero to a tested, CI-deployed AWS project. Each part builds on the previous one.",
    pages: {
      slugs: [
        "aws/tutorial/part-1",
        "aws/tutorial/part-2",
        "aws/tutorial/part-3",
        "aws/tutorial/part-4",
        "aws/tutorial/part-5",
      ],
    },
  },
  {
    heading: "AWS — Compute",
    pages: { directory: "aws/compute", exclude: ["aws/compute/choosing-a-runtime"] },
  },
  {
    heading: "AWS — Frontend",
    pages: { directory: "aws/frontend" },
  },
  {
    heading: "AWS — APIs",
    pages: { directory: "aws/apis" },
  },
  {
    heading: "AWS — Data",
    pages: { directory: "aws/data" },
  },
  {
    heading: "AWS — AI",
    pages: { directory: "aws/ai" },
  },
  {
    heading: "AWS — Messaging & events",
    pages: { directory: "aws/messaging" },
  },
  {
    heading: "AWS — Email",
    pages: { directory: "aws/email" },
  },
  {
    heading: "AWS — Security & secrets",
    pages: { directory: "aws/security" },
  },
  {
    heading: "AWS — Observability",
    pages: { directory: "aws/observability" },
  },
  {
    heading: "AWS — Networking",
    pages: { directory: "aws/networking" },
  },
  {
    heading: "GCP",
    intro:
      "Google Cloud as Stack resources — Cloud Run services and jobs, Firestore, Pub/Sub, BigQuery, Secret Manager, Memorystore. Bindings grant IAM onto a per-host service account.",
    pages: {
      slugs: ["gcp/index", "gcp/setup"],
    },
  },
  {
    heading: "GCP — Tutorial",
    pages: { directory: "gcp/tutorial" },
  },
  {
    heading: "GCP — Compute",
    pages: { directory: "gcp/compute" },
  },
  {
    heading: "GCP — Frontend",
    pages: { directory: "gcp/frontend" },
  },
  {
    heading: "GCP — APIs",
    pages: { directory: "gcp/apis" },
  },
  {
    heading: "GCP — Data",
    pages: { directory: "gcp/data" },
  },
  {
    heading: "GCP — AI",
    pages: { directory: "gcp/ai" },
  },
  {
    heading: "GCP — Messaging & events",
    pages: { directory: "gcp/messaging" },
  },
  {
    heading: "GCP — Security & secrets",
    pages: { directory: "gcp/security" },
  },
  {
    heading: "GCP — Observability",
    pages: { directory: "gcp/observability" },
  },
  {
    heading: "GCP — Networking",
    pages: { directory: "gcp/networking" },
  },
  {
    heading: "Fly — start here",
    intro:
      "The Fly hub: overview (Apps, Machines, Services, Sprites), setup (org, API token, profile), and networking.",
    pages: {
      slugs: ["fly/index", "fly/setup", "fly/networking"],
    },
  },
  {
    heading: "Fly — Tutorial",
    pages: { directory: "fly/tutorial" },
  },
  {
    heading: "Fly — Compute",
    pages: { directory: "fly/compute" },
  },
  {
    heading: "Fly — Data",
    pages: { directory: "fly/data" },
  },
  {
    heading: "Fly — Frontend",
    pages: { directory: "fly/frontend" },
  },
  {
    heading: "Hetzner — start here",
    intro:
      "The Hetzner hub: overview (Servers and Services) and setup (project, API token, profile).",
    pages: {
      slugs: ["hetzner/index", "hetzner/setup"],
    },
  },
  {
    heading: "Hetzner — Tutorial",
    pages: { directory: "hetzner/tutorial" },
  },
  {
    heading: "Hetzner — Compute",
    pages: { directory: "hetzner/compute" },
  },
  {
    heading: "Hetzner — Data",
    pages: { directory: "hetzner/data" },
  },
  {
    heading: "Hetzner — Networking",
    pages: { directory: "hetzner/networking" },
  },
  {
    heading: "Hetzner — Frontend",
    pages: { directory: "hetzner/frontend" },
  },
  {
    heading: "Railway — start here",
    intro:
      "The Railway hub: overview (Projects, Services, databases), setup (workspace, API token, profile), and networking.",
    pages: {
      slugs: ["railway/index", "railway/setup", "railway/networking"],
    },
  },
  {
    heading: "Railway — Tutorial",
    pages: { directory: "railway/tutorial" },
  },
  {
    heading: "Railway — Compute",
    pages: { directory: "railway/compute" },
  },
  {
    heading: "Railway — Data",
    pages: { directory: "railway/data" },
  },
  {
    heading: "Railway — Frontend",
    pages: { directory: "railway/frontend" },
  },
  {
    heading: "PlanetScale",
    intro:
      "Serverless MySQL & Postgres as Stack resources. Composes with Cloudflare Hyperdrive and Drizzle — those guides are listed under Cloudflare.",
    pages: {
      slugs: [
        "planetscale/index",
        "planetscale/setup",
        "planetscale/data/postgres",
        "planetscale/data/mysql",
        "planetscale/data/migrations",
        "planetscale/data/credentials",
        "planetscale/data/backups",
        "planetscale/guides/preview-branches",
        "planetscale/guides/drizzle",
      ],
    },
  },
  {
    heading: "Neon",
    intro:
      "Neon backend resources: Postgres branches, Functions, private object storage, managed Auth, Data API, AI Gateway and Website adapters. The upload tutorial combines native and Effect handlers with authenticated browser uploads.",
    pages: {
      slugs: [
        "neon/index",
        "neon/setup",
        "neon/governance",
        "neon/guides/ai-gateway",
        "neon/guides/production-auth",
        "neon/guides/private-networking",
        "neon/guides/custom-domains",
        "neon/guides/state-recovery",
        "neon/tutorial/index",
        "neon/tutorial/backend",
        "neon/tutorial/functions",
        "neon/tutorial/frontend",
        "neon/tutorial/previews",
        "neon/data/branching",
        "neon/data/connections",
        "neon/data/roles",
        "neon/data/migrations",
        "neon/guides/preview-branches",
        "neon/guides/drizzle",
      ],
    },
  },
  {
    heading: "Neon — Frontend",
    pages: { directory: "neon/frontend" },
  },
  {
    heading: "Turso",
    intro:
      "Turso SQLite databases as Stack resources: groups, databases, migrations, connections, a database per tenant, preview databases, and scoped tokens.",
    pages: {
      slugs: [
        "turso/index",
        "turso/setup",
        "turso/tutorial/part-1",
        "turso/tutorial/part-2",
        "turso/tutorial/part-3",
        "turso/groups",
        "turso/databases",
        "turso/migrations",
        "turso/connections",
        "turso/database-per-tenant",
        "turso/preview-databases",
        "turso/tokens",
        "turso/organization",
      ],
    },
  },
  {
    heading: "Better Auth",
    intro:
      "Better Auth as an Effect service on Workers and Lambda: a tutorial, sign-in providers, database layers, and guides for config, middleware, secondary storage, migrations, and upgrades.",
    pages: { slugs: ["better-auth/index"] },
  },
  {
    heading: "Better Auth — Tutorial",
    pages: { directory: "better-auth/tutorial" },
  },
  {
    heading: "Better Auth — Sign-in providers",
    pages: { directory: "better-auth/sign-in-providers" },
  },
  {
    heading: "Better Auth — Databases",
    pages: { directory: "better-auth/databases" },
  },
  {
    heading: "Better Auth — Guides",
    pages: {
      slugs: [
        "better-auth/guides/configuration",
        "better-auth/guides/http-api-middleware",
        "better-auth/guides/secondary-storage",
        "better-auth/guides/migrations",
        "better-auth/upgrades/from-1-6-to-1-7",
      ],
    },
  },
  {
    heading: "Axiom",
    intro:
      "Observability as Stack resources — datasets, monitors, notifiers. The exporter-Layer pattern is documented in Concepts → Observability.",
    pages: {
      slugs: [
        "axiom/index",
        "axiom/setup",
        "axiom/data/ingest",
        "axiom/guides/alerting",
        "axiom/guides/dashboards",
        "axiom/guides/annotations",
      ],
    },
  },
  {
    heading: "GitHub",
    intro:
      "Repos, secrets, variables, and repository event sources as Stack resources; the CI/CD guides live under Guides and each cloud's tutorial part 5.",
    pages: {
      slugs: [
        "github/index",
        "github/setup",
        "github/repository",
        "github/actions-config",
        "github/events",
      ],
    },
  },
  {
    heading: "Git",
    intro:
      "A git server you build in one file, use like any remote, and reshape one block at a time: setup, a six-part tutorial (push, access control, publishing, per-user credentials, your app's API, branch protection), the REST and GitHub-compatible planes, the building blocks with their implementations, and recipes with the scaling model and measurements.",
    pages: {
      slugs: [
        "git/index",
        "git/getting-started",
        "git/tutorial/part-1",
        "git/tutorial/part-2",
        "git/tutorial/part-3",
        "git/tutorial/part-4",
        "git/tutorial/part-5",
        "git/tutorial/part-6",
        "git/clone-and-push",
        "git/repositories",
        "git/pull-requests",
        "git/github-api",
        "git/blocks/index",
        "git/blocks/server",
        "git/blocks/engine",
        "git/blocks/repositories",
        "git/blocks/registry",
        "git/blocks/blob-store",
        "git/blocks/hasher",
        "git/blocks/auth",
        "git/recipes/index",
        "git/recipes/scaling",
        "git/recipes/cloudflare",
        "git/recipes/cloudflare-aws",
        "git/recipes/your-own-store",
      ],
    },
  },
  {
    heading: "Stripe",
    intro:
      "Products, prices, coupons, customers, payment links, and webhooks as Stack resources. HTTP bindings run inside a Worker or Lambda.",
    pages: {
      slugs: [
        "stripe/index",
        "stripe/setup",
        "stripe/guides/subscriptions",
        "stripe/guides/connect",
        "stripe/guides/webhooks",
      ],
    },
  },
  {
    heading: "Docker",
    intro:
      "Local and CI Docker as Stack resources — images, containers, networks, and volumes driven through the active Docker CLI context; cloud container runtimes (Cloudflare Containers, ECS) consume the pushed image refs from their own hubs.",
    pages: {
      slugs: [
        "docker/index",
        "docker/setup",
        "docker/local-services",
        "docker/build-and-push",
        "docker/swarm",
      ],
    },
  },
  {
    heading: "Kubernetes",
    intro:
      "Cluster-agnostic Kubernetes workloads — Deployments, Jobs and CronJobs (as images or Effect programs), raw manifests, and Helm charts — on a local cluster or any cluster reachable through a kubeconfig, token, client certificate, or exec plugin.",
    pages: {
      slugs: [
        "kubernetes/index",
        "kubernetes/setup",
        "kubernetes/tutorial/part-1",
        "kubernetes/tutorial/part-2",
        "kubernetes/tutorial/part-3",
        "kubernetes/tutorial/part-4",
        "kubernetes/tutorial/part-5",
        "kubernetes/clusters/connecting",
        "kubernetes/clusters/registries",
        "kubernetes/clusters/local",
        "kubernetes/clusters/eks",
        "kubernetes/clusters/cluster-adapters",
        "kubernetes/workloads/deployments",
        "kubernetes/workloads/jobs",
        "kubernetes/workloads/images",
        "kubernetes/workloads/bindings",
        "kubernetes/workloads/object-lifecycle",
        "kubernetes/objects/manifests",
        "kubernetes/objects/helm-charts",
      ],
    },
  },
  {
    heading: "SQL",
    intro:
      "Database comparison, Effect SQL, Drizzle, Prisma ORM v8 (Postgres-only), committed migrations, and connection lifecycle. Deployment walkthroughs live with their runtime providers.",
    pages: {
      slugs: [
        "sql/index",
        "sql/databases",
        "sql/effect-sql/postgres",
        "sql/effect-sql/mysql",
        "sql/effect-sql/d1",
        "sql/effect-sql/libsql",
        "sql/effect-sql/migrations",
        "sql/effect-sql/lifecycle",
        "sql/drizzle/postgres",
        "sql/drizzle/mysql",
        "sql/drizzle/d1",
        "sql/drizzle/libsql",
        "sql/drizzle/migrations",
        "sql/prisma/postgres",
        "sql/prisma/contracts",
        "sql/prisma/migrations",
      ],
    },
  },
  {
    heading: "Command",
    intro:
      "Cloud-agnostic local process primitives — memoized builds, one-off commands, and dev servers; the static-site guides in each cloud hub consume them under the hood.",
    pages: {
      slugs: ["command/index", "command/memoization", "command/dev-servers"],
    },
  },
  {
    heading: "ACME",
    intro:
      "TLS certificates from any ACME certificate authority as Stack resources, validated over DNS, renewed on deploy, or issued at runtime.",
    pages: {
      slugs: [
        "acme/index",
        "acme/getting-started",
        "acme/certificate-authorities",
        "acme/dns-validation",
        "acme/renewal",
        "acme/runtime",
        "acme/using-certificates",
        "acme/troubleshooting",
      ],
    },
  },
  {
    heading: "Domain",
    intro:
      "A domain-driven design framework: model commands, events, aggregates, views, projections, policies, and sagas once, then host them on Durable Objects or in memory.",
    pages: {
      slugs: [
        "domain/index",
        "domain/what-is-ddd",
        "domain/project-layout",
        "domain/tutorial/part-1",
        "domain/tutorial/part-2",
        "domain/tutorial/part-3",
        "domain/tutorial/part-4",
        "domain/tutorial/part-5",
        "domain/tutorial/part-6",
        "domain/tutorial/part-7",
        "domain/tutorial/part-8",
        "domain/model",
        "domain/command",
        "domain/event",
        "domain/aggregate",
        "domain/view",
        "domain/projection",
        "domain/stream",
        "domain/policy",
        "domain/saga",
        "domain/external-system",
        "domain/api",
        "domain/story",
        "domain/testing-models",
        "domain/testing-apis",
        "domain/foldkit",
        "domain/optimistic-updates",
        "domain/local-first",
        "domain/mcp",
        "domain/sessions",
        "domain/better-auth",
        "domain/view-security",
        "domain/unique-values",
        "domain/deadlines",
        "domain/cloudflare",
        "domain/backfill",
        "domain/in-memory",
        "domain/http-api",
        "domain/cli",
        "domain/observability",
      ],
    },
  },
];

const PROVIDERS_INTRO_BASE = `Service-level API reference, generated from JSDoc on the source \`.ts\` files via \`bun scripts/generate-api-reference.ts\`. Each page groups resources and capabilities by service, with a heading per resource and examples extracted from source JSDoc.`;

const PROVIDERS_INTRO = `${PROVIDERS_INTRO_BASE} Grouped by cloud below.`;

/**
 * Enumerates every generated provider page under `providers/{Cloud}/...`,
 * grouped by cloud. These pages are produced by `build:reference` (which runs
 * before this script in the build), so they exist on disk at generation time
 * even though they are gitignored.
 */
async function renderProvidersSection(): Promise<string> {
  const providersDir = path.join(docsDir, "providers");
  let clouds: string[];
  try {
    const entries = await readdir(providersDir, { withFileTypes: true });
    clouds = entries
      .filter((e) => e.isDirectory())
      .map((e) => e.name)
      .sort();
  } catch (err: any) {
    if (err?.code === "ENOENT") return `## Providers\n\n${PROVIDERS_INTRO}`;
    throw err;
  }

  const blocks: string[] = [`## Providers`, PROVIDERS_INTRO];
  for (const cloud of clouds) {
    const slugs = await listSlugs(`providers/${cloud}`);
    const pages = (await Promise.all(slugs.map(loadPage)))
      .filter((p) => !p.draft)
      // Starlight serves provider routes lowercased (e.g. the CamelCase source
      // `providers/AWS/S3/Bucket.md` is reachable at `/providers/aws/s3/bucket`).
      .map((p) => ({ ...p, href: p.href.toLowerCase() }))
      .sort((a, b) => a.title.localeCompare(b.title));
    if (pages.length === 0) continue;
    blocks.push(`### ${cloud}`);
    blocks.push(pages.map(renderPage).join("\n"));
  }
  return blocks.join("\n\n");
}

const HEADER = `# Alchemy

> Alchemy Effect is an Infrastructure-as-Effects (IaE) framework that combines cloud infrastructure and application logic into a single, type-safe program powered by [Effect](https://effect.website). Resources are declared as Effects; bindings wire IAM, env vars, and typed SDKs in one call; deploys and runtime share the same code.

This file is a navigation index for the documentation site at ${siteUrl}. Guide and concept pages are listed below with their URL and a one-line summary, so an agent can pick the right page in one hop. The service-level API reference is indexed separately in [llms-full.txt](${siteUrl}/llms-full.txt).`;

const FULL_HEADER = `# Alchemy

> Alchemy Effect is an Infrastructure-as-Effects (IaE) framework that combines cloud infrastructure and application logic into a single, type-safe program powered by [Effect](https://effect.website). Resources are declared as Effects; bindings wire IAM, env vars, and typed SDKs in one call; deploys and runtime share the same code.

This file is a navigation index for the documentation site at ${siteUrl}. Every page under \`/src/content/docs/\` is listed below with its URL and a one-line summary, so an agent can pick the right page in one hop. This is the full index including every service-level API reference page; the smaller [llms.txt](${siteUrl}/llms.txt) covers just the guides and concept docs.`;

function parseFrontmatter(source: string): Record<string, string> {
  if (!source.startsWith("---")) return {};
  const end = source.indexOf("\n---", 3);
  if (end === -1) return {};
  const block = source.slice(3, end);
  const out: Record<string, string> = {};
  for (const rawLine of block.split("\n")) {
    const line = rawLine.trimEnd();
    const m = line.match(/^([A-Za-z_][A-Za-z0-9_-]*):\s*(.*)$/);
    if (!m) continue;
    let value = m[2].trim();
    if (
      (value.startsWith('"') && value.endsWith('"')) ||
      (value.startsWith("'") && value.endsWith("'"))
    ) {
      value = value.slice(1, -1);
    }
    out[m[1]] = value;
  }
  return out;
}

/**
 * Extracts the nested `sidebar.order` value from a frontmatter block.
 * Starlight uses this to order autogenerated sidebar groups; we mirror it
 * so llms.txt lists pages in the same order the sidebar shows them.
 * Returns `Infinity` when unset, so unordered pages sort after ordered ones.
 */
function parseSidebarOrder(source: string): number {
  if (!source.startsWith("---")) return Number.POSITIVE_INFINITY;
  const end = source.indexOf("\n---", 3);
  if (end === -1) return Number.POSITIVE_INFINITY;
  const block = source.slice(3, end);
  const lines = block.split("\n");
  for (let i = 0; i < lines.length; i++) {
    if (!/^sidebar:\s*$/.test(lines[i])) continue;
    for (let j = i + 1; j < lines.length; j++) {
      if (/^\S/.test(lines[j])) break; // dedented out of the sidebar block
      const m = lines[j].match(/^\s+order:\s*(-?[\d.]+)\s*$/);
      if (m) return Number.parseFloat(m[1]);
    }
    break;
  }
  return Number.POSITIVE_INFINITY;
}

async function loadPage(slug: string): Promise<Page> {
  const candidates = [`${slug}.mdx`, `${slug}.md`];
  for (const rel of candidates) {
    const full = path.join(docsDir, rel);
    try {
      // Dev-generated reference pages carry copy-editor markers.
      const source = (await readFile(full, "utf8")).replace(
        /<!--copy:[^>]*?-->|^<div data-copy=[^\n]*>\n|^<\/div><!--\/copy-->\n/gm,
        "",
      );
      const fm = parseFrontmatter(source);
      const title = fm.title;
      const description = fm.description ?? fm.excerpt ?? "";
      if (!title) {
        throw new Error(`Missing title in frontmatter: ${rel}`);
      }
      return {
        // `foo/index.mdx` is served at `/foo`.
        href: `/${slug}`.replace(/\/index$/, "") || "/",
        slug,
        title,
        description,
        draft: fm.draft === "true" || (fm.draft as unknown as boolean) === true,
        order: parseSidebarOrder(source),
      };
    } catch (err: any) {
      if (err?.code !== "ENOENT") throw err;
    }
  }
  throw new Error(`Page not found: ${slug} (looked for .mdx and .md)`);
}

async function listSlugs(directory: string, exclude: string[] = []): Promise<string[]> {
  const dir = path.join(docsDir, directory);
  const entries = await readdir(dir, { withFileTypes: true });
  const slugs: string[] = [];
  for (const entry of entries) {
    const rel = `${directory}/${entry.name}`;
    if (entry.isDirectory()) {
      slugs.push(...(await listSlugs(rel, exclude)));
      continue;
    }
    if (!entry.isFile()) continue;
    const ext = path.extname(entry.name);
    if (ext !== ".md" && ext !== ".mdx") continue;
    const slug = `${directory}/${entry.name.slice(0, -ext.length)}`;
    if (exclude.includes(slug)) continue;
    slugs.push(slug);
  }
  slugs.sort();
  return slugs;
}

function byOrderThenTitle(a: Page, b: Page): number {
  if (a.order !== b.order) return a.order - b.order;
  return a.title.localeCompare(b.title);
}

function renderPage(page: Page): string {
  const url = `${siteUrl}${page.href}`;
  const desc = page.description ? ` — ${page.description}` : "";
  return `- [${page.title}](${url})${desc}`;
}

/**
 * Fails when a published page is missing from SECTIONS or listed twice, so
 * a new docs page can never silently drop out of llms.txt. The generated
 * API reference (`providers/`) and the blog are indexed separately.
 */
async function assertCoverage(listed: string[]) {
  const counts = new Map<string, number>();
  for (const slug of listed) counts.set(slug, (counts.get(slug) ?? 0) + 1);
  const duplicates = [...counts].filter(([, n]) => n > 1).map(([slug]) => slug);

  const entries = await readdir(docsDir, { withFileTypes: true });
  const published: string[] = [];
  for (const entry of entries) {
    if (entry.name === "providers" || entry.name === "blog") continue;
    const slugs = entry.isDirectory()
      ? await listSlugs(entry.name)
      : [".md", ".mdx"].includes(path.extname(entry.name))
        ? [entry.name.slice(0, -path.extname(entry.name).length)]
        : [];
    for (const page of await Promise.all(slugs.map(loadPage))) {
      if (!page.draft) published.push(page.slug);
    }
  }
  const missing = published.filter((slug) => !counts.has(slug));

  const problems = [
    ...missing.map((slug) => `not listed in any SECTIONS entry: ${slug}`),
    ...duplicates.map((slug) => `listed in more than one SECTIONS entry: ${slug}`),
  ];
  if (problems.length > 0) {
    throw new Error(
      `llms.txt does not match the docs:\n  ${problems.join("\n  ")}\n` +
        `Add each page to a section in scripts/generate-llms-txt.ts.`,
    );
  }
}

async function main() {
  const sections: string[] = [];
  const listed: string[] = [];

  for (const section of SECTIONS) {
    const isDirectory = !("slugs" in section.pages);
    const slugs =
      "slugs" in section.pages
        ? section.pages.slugs
        : await listSlugs(section.pages.directory, section.pages.exclude);
    listed.push(...slugs);
    const pages = (await Promise.all(slugs.map(loadPage))).filter((p) => !p.draft);
    // Directory sections mirror the sidebar's `sidebar.order` ordering; slug
    // sections keep the curated order they were declared in.
    if (isDirectory) pages.sort(byOrderThenTitle);

    sections.push(`## ${section.heading}`);
    if (section.intro) sections.push(section.intro);
    sections.push(pages.map(renderPage).join("\n"));
  }

  await assertCoverage(listed);

  const fullBody = [FULL_HEADER, ...sections, await renderProvidersSection()].join("\n\n") + "\n";
  await writeFile(outFullFile, fullBody, "utf8");
  console.log(`Wrote ${outFullFile}`);

  const fullSizeKb = Math.round(Buffer.byteLength(fullBody, "utf8") / 1024);
  const referenceSection = [
    `## API reference`,
    `${PROVIDERS_INTRO_BASE}\n\nThe service-level pages are indexed in [llms-full.txt](${siteUrl}/llms-full.txt), which repeats this entire file plus one link per service. Warning: it is large (~${fullSizeKb} KB) — only fetch it when you need to locate a specific resource's reference page.`,
  ];

  const body = [HEADER, ...sections, ...referenceSection].join("\n\n") + "\n";
  await writeFile(outFile, body, "utf8");
  console.log(`Wrote ${outFile}`);
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});