import type { WorkerEnv } from "../alchemy.run.ts";
import { referenceDestination } from "./reference-links.ts";
import { SOCIAL_REDIRECTS } from "./social-redirects.ts";
// Minimal `HTMLRewriter` shape — the workers runtime exposes it as a
// global, but we don't pull in `@cloudflare/workers-types`, so declare
// just what this file uses.
declare class HTMLRewriter {
on(selector: string, handler: { element(el: HTMLRewriterElement): void }): HTMLRewriter;
transform(response: Response): Response;
}
interface HTMLRewriterElement {
getAttribute(name: string): string | null;
setAttribute(name: string, value: string): HTMLRewriterElement;
}
/**
* `https://alchemy.run` is the one indexable origin. Every other host this
* worker answers on — `main.alchemy.run`, PR previews on `*.workers.dev`,
* future aliases — is a mirror of the same build:
*
* - `<link rel="canonical">` is baked at build time against `site` and is left
* alone, so mirrors point search engines back at production.
* - Every mirror response carries `X-Robots-Tag: noindex` and its `robots.txt`
* advertises no sitemap, so a mirror URL that leaks (a PR comment, a shared
* link) never becomes a duplicate search result.
* - `og:*` / `twitter:*` card URLs ARE rewritten to the request host, so a
* Slack/X unfurl of a preview URL shows that preview's card, not
* production's.
*/
const CANONICAL_HOST = "alchemy.run";
const CANONICAL_ORIGIN = `https://${CANONICAL_HOST}`;
/**
* The v1 docs moved to their own host. v1-era URLs (`/docs/...`, and the
* pre-hub `/guides/...` / `/providers/...` / `/concepts/...` trees) are still
* in search indexes; instead of 404ing them we probe v1 and 301 when it has
* the page.
*/
const V1_ORIGIN = "https://v1.alchemy.run";
const V1_PREFIXES = ["/docs/", "/guides/", "/providers/", "/concepts/"];
/**
* 301s for the docs restructure (guides/tutorials moved into per-cloud hubs).
* Keys and targets are extensionless; `.md` requests (agents fetching raw
* markdown) are redirected to the target's `.md` form with any fragment
* dropped.
*/
const REDIRECTS: Record<string, string> = {
"/tutorial/part-1": "/cloudflare/tutorial/part-1",
"/tutorial/part-2": "/cloudflare/tutorial/part-2",
"/tutorial/part-3": "/cloudflare/tutorial/part-3",
"/tutorial/part-4": "/cloudflare/tutorial/part-4",
"/tutorial/part-5": "/cloudflare/tutorial/part-5",
"/tutorial/cloudflare/compute/durable-objects": "/cloudflare/compute/durable-objects",
"/tutorial/cloudflare/data/hyperdrive": "/cloudflare/data/hyperdrive",
"/tutorial/cloudflare/queue-consumer": "/cloudflare/messaging/queues",
"/tutorial/cloudflare/rpc-durable-object": "/cloudflare/compute/durable-objects#schemaless-rpc",
"/tutorial/cloudflare/rpc-worker": "/cloudflare/compute/workers#call-another-worker",
"/tutorial/cloudflare/ai-gateway": "/cloudflare/ai/ai-gateway",
"/tutorial/cloudflare/ai-search": "/cloudflare/ai/ai-search",
"/tutorial/cloudflare/artifacts": "/cloudflare/data/artifacts",
"/tutorial/cloudflare/branch-from-shared-database":
"/cloudflare/data/branch-from-shared-database",
"/tutorial/cloudflare/compute/containers": "/cloudflare/compute/run-a-container",
"/tutorial/cloudflare/cross-worker-durable-object":
"/cloudflare/compute/cross-worker-durable-object",
"/tutorial/cloudflare/drizzle": "/cloudflare/data/drizzle",
"/tutorial/cloudflare/hibernatable-websockets": "/cloudflare/compute/hibernatable-websockets",
"/tutorial/cloudflare/vite-spa": "/cloudflare/frontend/vite-spa",
"/tutorial/cloudflare/compute/workflows": "/cloudflare/compute/add-a-workflow",
"/tutorial/aws/compute/lambda": "/aws/compute/lambda",
"/tutorial/aws/data/dynamodb": "/aws/data/dynamodb",
"/tutorial/aws/messaging/sqs": "/aws/messaging/sqs",
"/tutorial/aws/data/s3": "/aws/data/s3",
"/tutorial/aws/messaging/kinesis": "/aws/messaging/kinesis",
"/tutorial/aws/api-gateway": "/aws/apis/api-gateway",
"/tutorial/aws/dynamodb-streams": "/aws/messaging/dynamodb-streams",
"/tutorial/aws/s3-events": "/aws/messaging/s3-events",
"/guides/effect-http-api": "/cloudflare/apis/effect-http-api",
"/guides/effect-rpc": "/cloudflare/apis/effect-rpc",
"/guides/effect-ai": "/cloudflare/ai/effect-ai",
"/guides/frontends": "/cloudflare/frontend/frontends",
"/guides/shared-database": "/cloudflare/data/shared-database",
// Absorbed into the Security & secrets block pages.
"/guides/secrets": "/cloudflare/security/secrets-env",
"/guides/stack-references": "/infrastructure-as-code/references",
// The CLI reference became its own hub tab.
"/guides/cli": "/cli",
"/aws/guides/secrets": "/aws/security/secrets-env",
// The Integrations tab was replaced by per-provider hubs (PlanetScale and
// Neon promoted to top-level tabs; Axiom and GitHub in the More menu).
// Section-folder alignment: file locations now mirror the sidebar taxonomy.
"/aws/choosing-a-runtime": "/aws/compute/choosing-a-runtime",
"/aws/cloudwatch": "/aws/observability/cloudwatch",
"/aws/dynamodb": "/aws/data/dynamodb",
"/aws/ec2": "/aws/compute/ec2",
"/aws/ecs": "/aws/compute/ecs",
"/aws/eks": "/aws/compute/eks",
"/aws/eventbridge": "/aws/messaging/eventbridge",
"/aws/guides/api-gateway": "/aws/apis/api-gateway",
"/aws/guides/custom-domains": "/aws/networking/custom-domains",
"/aws/guides/dynamodb-streams": "/aws/messaging/dynamodb-streams",
"/aws/guides/effect-http-api": "/aws/apis/effect-http-api",
"/aws/guides/effect-rpc": "/aws/apis/effect-rpc",
"/aws/guides/microvms": "/aws/compute/microvms",
"/aws/guides/s3-events": "/aws/messaging/s3-events",
"/aws/guides/static-site": "/aws/frontend/static-site",
"/aws/kinesis": "/aws/messaging/kinesis",
"/aws/lambda": "/aws/compute/lambda",
"/aws/rds": "/aws/data/rds",
"/aws/s3": "/aws/data/s3",
"/aws/secrets-env": "/aws/security/secrets-env",
"/aws/sns": "/aws/messaging/sns",
"/aws/sqs": "/aws/messaging/sqs",
"/aws/websites": "/aws/frontend/websites",
"/axiom/ingest": "/axiom/data/ingest",
"/cloudflare/containers": "/cloudflare/compute/containers",
"/cloudflare/d1": "/cloudflare/data/d1",
"/cloudflare/domains": "/cloudflare/networking/domains",
"/cloudflare/durable-objects": "/cloudflare/compute/durable-objects",
"/cloudflare/guides/ai-gateway": "/cloudflare/ai/ai-gateway",
"/cloudflare/guides/ai-search": "/cloudflare/ai/ai-search",
"/cloudflare/guides/analytics-engine": "/cloudflare/observability/analytics-engine",
"/cloudflare/guides/artifacts": "/cloudflare/data/artifacts",
"/cloudflare/guides/axiom-observability": "/cloudflare/observability/axiom-observability",
"/cloudflare/guides/branch-from-shared-database": "/cloudflare/data/branch-from-shared-database",
"/cloudflare/guides/browser-rendering": "/cloudflare/compute/browser-rendering",
"/cloudflare/guides/containers": "/cloudflare/compute/run-a-container",
"/cloudflare/guides/cron": "/cloudflare/messaging/cron",
"/cloudflare/guides/cross-worker-durable-object":
"/cloudflare/compute/cross-worker-durable-object",
"/cloudflare/guides/custom-domains": "/cloudflare/networking/custom-domains",
"/cloudflare/guides/drizzle": "/cloudflare/data/drizzle",
"/cloudflare/guides/effect-ai": "/cloudflare/ai/effect-ai",
"/cloudflare/guides/effect-http-api": "/cloudflare/apis/effect-http-api",
"/cloudflare/guides/effect-rpc": "/cloudflare/apis/effect-rpc",
"/cloudflare/guides/email": "/cloudflare/email/send-and-receive",
"/cloudflare/guides/frontends": "/cloudflare/frontend/frontends",
"/cloudflare/guides/github-events": "/cloudflare/messaging/github-events",
"/cloudflare/guides/hibernatable-websockets": "/cloudflare/compute/hibernatable-websockets",
"/cloudflare/guides/secrets-store": "/cloudflare/security/secrets-store",
"/cloudflare/guides/shared-database": "/cloudflare/data/shared-database",
"/cloudflare/guides/tunnel": "/cloudflare/networking/tunnel",
"/cloudflare/guides/turnstile": "/cloudflare/security/turnstile",
"/cloudflare/guides/vectorize": "/cloudflare/ai/vectorize",
"/cloudflare/guides/vite-spa": "/cloudflare/frontend/vite-spa",
"/cloudflare/guides/workers-for-platforms": "/cloudflare/compute/workers-for-platforms",
"/cloudflare/guides/workflows": "/cloudflare/compute/add-a-workflow",
"/cloudflare/hyperdrive": "/cloudflare/data/hyperdrive",
"/cloudflare/kv": "/cloudflare/data/kv",
"/cloudflare/queues": "/cloudflare/messaging/queues",
"/cloudflare/r2": "/cloudflare/data/r2",
"/cloudflare/secrets-env": "/cloudflare/security/secrets-env",
"/cloudflare/workers": "/cloudflare/compute/workers",
"/cloudflare/workflows": "/cloudflare/compute/workflows",
"/concepts/action": "/infrastructure-as-code/action",
"/concepts/binding": "/infrastructure-as-effects/binding",
"/concepts/layers": "/infrastructure-as-effects/layers",
"/concepts/local-development": "/environments/local-development",
"/concepts/observability": "/testing/observability",
"/concepts/outputs": "/infrastructure-as-code/outputs",
"/concepts/phases": "/infrastructure-as-effects/phases",
"/concepts/platform": "/infrastructure-as-effects/runtime",
"/infrastructure-as-effects/platform": "/infrastructure-as-effects/runtime",
"/infrastructure-as-effects/functions-and-servers": "/infrastructure-as-effects/runtime",
"/rpc": "/apis",
"/rpc/schemaless": "/apis/schemaless",
"/rpc/effect-rpc": "/apis/effect-rpc",
"/rpc/effect-http": "/apis/effect-http",
"/concepts/profiles": "/environments/profiles",
"/concepts/provider": "/infrastructure-as-code/provider",
"/concepts/references": "/infrastructure-as-code/references",
"/concepts/resource": "/infrastructure-as-code/resource",
"/concepts/resource-lifecycle": "/infrastructure-as-code/resource-lifecycle",
"/concepts/secrets": "/environments/secrets",
"/concepts/stack": "/infrastructure-as-code/stack",
"/concepts/stages": "/environments/stages",
"/concepts/state-store": "/state-store",
"/concepts/test-harness": "/testing/test-harness",
"/concepts/testing": "/testing",
"/fly/frontend": "/fly/frontend/websites",
"/fly/websites": "/fly/frontend/websites",
"/gcp/guides/bindings": "/gcp/security/iam",
"/gcp/guides/cloud-run-api": "/gcp/compute/cloud-run",
"/gcp/guides/event-pipeline": "/gcp/messaging/pubsub",
"/gcp/guides/memorystore": "/gcp/data/memorystore",
"/guides/ci": "/environments/ci",
"/guides/circular-bindings": "/infrastructure-as-effects/circular-bindings",
"/guides/custom-provider": "/infrastructure-as-code/custom-provider",
"/guides/custom-state-store": "/state-store/custom-state-store",
"/guides/file-layout": "/project-structure/file-layout",
"/guides/infrastructure-layers": "/infrastructure-as-effects/layers",
"/infrastructure-as-effects/infrastructure-layers": "/infrastructure-as-effects/layers",
"/guides/migrating-from-v1": "/migrating-from-v1",
"/guides/monorepo": "/project-structure/monorepo",
"/guides/monorepo-multi-stack": "/project-structure/monorepo-multi-stack",
"/guides/monorepo-single-stack": "/project-structure/monorepo-single-stack",
"/guides/testing-a-stack": "/testing/testing-a-stack",
"/guides/testing-providers": "/testing/testing-providers",
"/hetzner/frontend": "/hetzner/frontend/websites",
"/hetzner/websites": "/hetzner/frontend/websites",
"/neon/branching": "/neon/data/branching",
"/neon/connections": "/neon/data/connections",
"/neon/migrations": "/neon/data/migrations",
"/planetscale/backups": "/planetscale/data/backups",
"/planetscale/credentials": "/planetscale/data/credentials",
"/planetscale/migrations": "/planetscale/data/migrations",
"/planetscale/mysql": "/planetscale/data/mysql",
"/planetscale/postgres": "/planetscale/data/postgres",
"/railway/frontend": "/railway/frontend/websites",
"/railway/websites": "/railway/frontend/websites",
"/integrations": "/providers",
"/integrations/planetscale": "/planetscale",
"/integrations/neon": "/neon",
"/integrations/axiom": "/axiom",
"/integrations/github": "/github",
"/git/auth": "/git/blocks/auth",
"/git/storage": "/git/blocks/repositories",
"/git/assemblies": "/git/recipes",
"/git/assemblies/cloudflare": "/git/recipes/cloudflare",
"/git/assemblies/cloudflare-aws": "/git/recipes/cloudflare-aws",
"/git/assemblies/one-origin": "/git/tutorial/part-4",
"/git/build": "/git/tutorial/part-4",
"/git/scale": "/git/recipes/scaling",
"/git/scale/pushes": "/git/blocks/hasher",
"/git/scale/clones": "/git/blocks/repositories",
};
const resolveRedirect = (url: URL): string | undefined => {
const social = SOCIAL_REDIRECTS[url.pathname.replace(/\/$/, "")];
if (social) return social;
const reference = referenceDestination(url.pathname + url.search);
if (reference) return reference;
let p = url.pathname.replace(/\/$/, "");
const isMarkdown = p.endsWith(".md");
if (isMarkdown) p = p.slice(0, -".md".length);
const target = REDIRECTS[p];
if (!target) return undefined;
if (isMarkdown) return `${target.split("#")[0]}.md`;
return target;
};
export default {
fetch: async (request: Request, env: WorkerEnv) => {
const url = new URL(request.url);
const canonical = url.host === CANONICAL_HOST;
const redirect = resolveRedirect(url);
if (redirect !== undefined) {
return Response.redirect(new URL(redirect, request.url), 301);
}
if (url.pathname === "/robots.txt" && !canonical) {
return new Response(MIRROR_ROBOTS_TXT, {
headers: {
"content-type": "text/plain; charset=utf-8",
"x-robots-tag": "noindex",
},
});
}
if (request.method === "GET" && prefersMarkdown(request)) {
const mdUrl = toMarkdownUrl(url).toString();
const res = await env.ASSETS.fetch(new Request(mdUrl, request));
// Astro's asset server labels `.md` as `application/octet-stream`, which
// agents treat as a binary download instead of rendering. Force the
// correct text type + charset since this branch only ever serves markdown.
if (res.status !== 404)
return withoutIndexing(
withContentType(res, "text/markdown; charset=utf-8"),
url,
canonical,
);
}
const res = await env.ASSETS.fetch(request);
if (res.status === 404 && request.method === "GET") {
const v1 = await resolveV1Fallback(url);
if (v1 !== undefined) return Response.redirect(v1, 301);
}
return withoutIndexing(
withUtf8Charset(await rewriteAgentTextOrigin(request, rewriteSocialCardHost(request, res))),
url,
canonical,
);
},
};
/**
* Served as `robots.txt` on every non-canonical host. Crawlable on purpose:
* a `Disallow` would stop crawlers from ever seeing the `noindex` header, and
* a blocked URL can still be indexed by URL alone. No `Sitemap:` line, so the
* mirror never advertises itself.
*/
const MIRROR_ROBOTS_TXT = `# Mirror of https://${CANONICAL_HOST} — every response is noindex.
User-agent: *
Allow: /
`;
/**
* Every page is mirrored as raw markdown (`/getting-started.md`, or the same
* URL requested with `Accept: text/markdown`) for agents. Search engines crawl
* those mirrors too and index them as separate documents — duplicating the HTML
* page and surfacing raw MDX (`import { Tabs } from "@astrojs/starlight/..."`)
* as the search result's snippet. `llms.txt` / `llms-full.txt` are the same
* kind of agent-facing text.
*
* Keep them crawlable (so the directive is actually seen) but out of the index.
* On a non-canonical host *everything* is noindex — the HTML included.
*/
const withoutIndexing = (res: Response, url: URL, canonicalHost: boolean): Response => {
const ct = res.headers.get("content-type") ?? "";
const agentText =
url.pathname.endsWith(".md") || ct.includes("text/markdown") || ct.includes("text/plain");
if (canonicalHost && !agentText) return res;
const next = new Response(res.body, res);
next.headers.set("x-robots-tag", "noindex");
return next;
};
/**
* v1-era URL that still resolves on the v1 docs host, or `undefined`. Only
* consulted after production 404s, so the extra HEAD subrequest is confined
* to dead links.
*/
const resolveV1Fallback = async (url: URL): Promise<string | undefined> => {
const pathname = url.pathname.replace(/\/$/, "");
if (pathname.endsWith(".md")) return undefined;
if (!V1_PREFIXES.some((prefix) => `${pathname}/`.startsWith(prefix))) {
return undefined;
}
const v1Path = pathname.startsWith("/docs/") ? pathname.slice("/docs".length) : pathname;
const target = `${V1_ORIGIN}${v1Path}${url.search}`;
try {
const probe = await fetch(target, { method: "HEAD", redirect: "follow" });
return probe.ok ? target : undefined;
} catch {
return undefined;
}
};
/**
* `llms.txt` / `llms-full.txt` are generated at build time with absolute
* canonical URLs, so a PR preview would hand agents an index that points back
* at production. Rewrite the baked origin to the request's own origin — the
* same treatment `rewriteSocialCardHost` gives the OG tags.
*/
const AGENT_TEXT_PATHS = new Set(["/llms.txt", "/llms-full.txt"]);
const rewriteAgentTextOrigin = async (request: Request, res: Response): Promise<Response> => {
const reqUrl = new URL(request.url);
if (!AGENT_TEXT_PATHS.has(reqUrl.pathname) || reqUrl.host === CANONICAL_HOST || !res.ok) {
return res;
}
const body = (await res.text()).replaceAll(`${CANONICAL_ORIGIN}/`, `${reqUrl.origin}/`);
return new Response(body, res);
};
/**
* Astro's static asset server labels `.txt` as `text/plain` with no charset.
* UTF-8 bytes (em dashes, arrows in our docs and `llms.txt`) then get decoded
* as latin-1 by browsers and agents, showing up as mojibake (`â€"`). Stamp
* `charset=utf-8` on text responses that omit it.
*/
const withUtf8Charset = (res: Response): Response => {
const ct = res.headers.get("content-type");
if (!ct || !ct.startsWith("text/") || /charset=/i.test(ct)) return res;
return withContentType(res, `${ct}; charset=utf-8`);
};
const withContentType = (res: Response, contentType: string): Response => {
const next = new Response(res.body, res);
next.headers.set("content-type", contentType);
return next;
};
/**
* Astro bakes absolute URLs into `og:image`, `og:url`, and `twitter:image`
* using the `site` config. Rewrite those to the request's host so each
* deployment unfurls itself. `<link rel="canonical">` is deliberately NOT
* rewritten — it must keep pointing at production on every mirror.
*/
const rewriteSocialCardHost = (request: Request, res: Response): Response => {
const ct = res.headers.get("content-type") ?? "";
if (!ct.includes("text/html")) return res;
const reqUrl = new URL(request.url);
if (reqUrl.host === CANONICAL_HOST) return res;
const content = {
element(el: HTMLRewriterElement) {
const value = el.getAttribute("content");
if (!value) return;
let u: URL;
try {
u = new URL(value);
} catch {
return;
}
if (u.host !== CANONICAL_HOST) return;
u.protocol = reqUrl.protocol;
u.host = reqUrl.host;
el.setAttribute("content", u.toString());
},
};
return new HTMLRewriter()
.on('meta[property="og:image"]', content)
.on('meta[property="og:url"]', content)
.on('meta[name="twitter:image"]', content)
.transform(res);
};
/**
* Returns true if the accept header prefers markdown or plain text over HTML.
*
* Examples:
* - opencode - accept: text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, *\/*;q=0.1
* - claude code - accept: application/json, text/plain, *\/*
*
* Notes:
* - ChatGPT and Claude web don't set an accept header; maybe check the user agent instead?
* - Cursor's headers are too generic (accept: *, user-agent: https://github.com/sindresorhus/got)
*/
const prefersMarkdown = (request: Request) => {
const accept = request.headers.get("accept");
if (!accept) return false;
// parse accept header and sort by quality; highest quality first
const types = accept
.split(",")
.map((part) => {
const type = part.split(";")[0].trim();
const q = part.match(/q=([^,]+)/)?.[1];
return { type, q: q ? Number.parseFloat(q) : 1 };
})
.sort((a, b) => b.q - a.q)
.map((type) => type.type);
const markdown = types.indexOf("text/markdown");
const plain = types.indexOf("text/plain");
const html = types.indexOf("text/html");
// if no HTML is specified, and either markdown or plain text is specified, prefer markdown
if (html === -1) {
return markdown !== -1 || plain !== -1;
}
// prefer markdown if higher quality than HTML
if ((markdown !== -1 && markdown < html) || (plain !== -1 && plain < html)) {
return true;
}
// otherwise, prefer HTML
return false;
};
function toMarkdownUrl(url: URL): URL {
const md = new URL(url.toString());
let p = md.pathname.replace(/\/$/, "");
if (p === "") p = "/index";
md.pathname = `${p}.md`;
return md;
}