import { dotAlchemyDirectory } from "alchemy/AlchemyContext";
import * as Artifacts from "alchemy/Artifacts";
import * as Bundle from "alchemy/Bundle/Bundle";
import { exec } from "alchemy/Util/exec";
import { isPathWithin } from "alchemy/Util/isPathWithin";
import { sha256 } from "alchemy/Util/sha256";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as ChildProcess from "effect/process/ChildProcess";
import path from "pathe";
import { convertPathToPattern, glob } from "tinyglobby";
import type { SourceContext, SourceProvider } from "../Source.ts";
import { bundleSource, resolveMainPath, watchBundleDirectory } from "./shared.ts";
/**
* Whether a Worker `main` entry points at a Python module. Python Workers
* skip the rolldown pipeline entirely — Cloudflare interprets `.py` sources
* directly with Pyodide, so the "bundle" is just the source files plus the
* vendored `python_modules/` directory.
*
* The parameter is `unknown` on purpose. `getCompatibility` reaches this from
* `WorkerProvider.precreate`, which is handed *raw, unresolved* props so
* resources in a dependency cycle can signal early — and `main` is an
* `Input`, so it can legitimately be an `Output` there (e.g. an entry path
* derived from a `Command.Build` output). An unresolved Output is a callable
* Proxy: it passes a `!== undefined` guard, `.split` yields another proxy,
* and calling it returns `undefined` — so a value-shaped check explodes with
* a bare `TypeError`. Narrowing on `typeof === "string"` keeps the predicate
* total: an unresolved `main` is simply not Python yet, and `reconcile`
* re-derives compatibility from fully resolved props.
*/
export const isPythonMain = (main: unknown): main is string =>
typeof main === "string" && main.split("?")[0].endsWith(".py");
/**
* The Pyodide cross-compilation targets uv understands, keyed by the Python
* version the Workers runtime selects from the compatibility date/flags.
* The wheel index does not have to match the exact Pyodide release the
* runtime uses — only the ABI (emscripten-wasm32) must be compatible.
* Mirrors pywrangler (cloudflare/workers-py `pywrangler/utils.py`).
*/
const PYODIDE_TARGETS = {
"3.12": {
interpreter: "cpython-3.12.7-emscripten-wasm32-musl",
index: "https://index.pyodide.org/0.27.7",
},
"3.13": {
interpreter: "cpython-3.13.2-emscripten-wasm32-musl",
index: "https://index.pyodide.org/0.28.3",
},
} as const;
// `python_workers_20250116` (Python 3.13 / Pyodide 0.28) is implied by
// `python_workers` for compatibility dates on or after this day.
const PYTHON_3_13_DEFAULT_ON = "2025-09-29";
/**
* The Python version the Workers runtime will use for the given
* compatibility settings — the same selection pywrangler performs.
*/
export const pythonVersionForCompatibility = (compatibility: {
date: string;
flags: string[];
}): keyof typeof PYODIDE_TARGETS => {
if (compatibility.flags.includes("no_python_workers_20250116")) {
return "3.12";
}
if (
compatibility.flags.includes("python_workers_20250116") ||
// ISO dates compare lexically.
compatibility.date >= PYTHON_3_13_DEFAULT_ON
) {
return "3.13";
}
return "3.12";
};
/**
* The PyPI package pywrangler always adds to the vendored dependencies: it
* provides the `workers` Python module (Request/Response/WorkerEntrypoint
* wrappers) plus the JS shims under `python_modules/workers/` that
* `import_from_javascript()` resolves.
*/
const MANAGED_SDK_PACKAGE = "workers-runtime-sdk";
export interface PythonWorkerBundleOptions {
id: string;
/** Namespace-qualified id — the display prefix for log lines. */
fqn: string;
/** Path (or `file://` URL) to the Worker's `.py` entry module. */
main: string;
compatibility: {
date: string;
flags: string[];
};
}
const uvError = (message: string) => (cause: unknown) =>
Bundle.BundleError.make({ message, cause });
/**
* Run a `uv` command, failing with a {@link Bundle.BundleError} that
* includes uv's output when the command exits non-zero, and with an
* actionable install hint when the binary is missing entirely.
*/
const runUv = Effect.fn(function* (
args: string[],
options: { cwd: string; env?: Record<string, string> },
) {
const result = yield* exec(
ChildProcess.make("uv", args, {
cwd: options.cwd,
env: options.env,
extendEnv: true,
}),
).pipe(
Effect.mapError(
uvError(
"Failed to run `uv` — Python Workers use uv (>= 0.8.10) to vendor " +
"dependencies for Pyodide. Install it from https://docs.astral.sh/uv/ " +
"and re-run.",
),
),
);
if (result.exitCode !== 0) {
return yield* Bundle.BundleError.make({
message: `\`uv ${args.join(" ")}\` failed with exit code ${result.exitCode}:\n${result.stdout}\n${result.stderr}`,
});
}
return result;
});
/**
* Resolve the `python_modules/` directory to upload alongside a Python
* Worker's sources:
*
* - If the Worker's directory already contains `python_modules/` (the
* user vendored it themselves, e.g. with `pywrangler sync` or any other
* tool that produces Wrangler's vendored layout), it is used as-is.
* - Otherwise, the managed Workers runtime SDK and any dependencies from an
* optional `pyproject.toml` are vendored with uv into a staging directory
* under `.alchemy/`:
* resolve against the Pyodide wheel index for the runtime's Python
* version (`uv pip compile` → `pylock.toml`), install the prebuilt
* wheels into an emscripten cross-venv (`uv venv` + `uv pip install
* --no-build`), and copy the venv's `site-packages` out. Re-vendoring
* is skipped while the `pyproject.toml` hash is unchanged.
*
* workerd no longer bundles the `workers` Python module at current
* compatibility dates, so even a dependency-free Worker needs the managed
* SDK under `python_modules/`.
*/
const resolvePythonModulesDir = Effect.fn(function* (
options: PythonWorkerBundleOptions & { root: string },
) {
const fs = yield* FileSystem.FileSystem;
const userManaged = path.join(options.root, "python_modules");
if (yield* orDefault(fs.exists(userManaged), false)) {
return userManaged;
}
const pyproject = path.join(options.root, "pyproject.toml");
const hasPyproject = yield* orDefault(fs.exists(pyproject), false);
const pythonVersion = pythonVersionForCompatibility(options.compatibility);
const target = PYODIDE_TARGETS[pythonVersion];
const pyprojectContent = hasPyproject
? yield* fs
.readFileString(pyproject)
.pipe(Effect.mapError(uvError(`Failed to read "${pyproject}"`)))
: "";
const inputHash = yield* sha256(`${pythonVersion}\0${target.index}\0${pyprojectContent}`);
const runtimeBase = process.cwd();
const dotAlchemy = yield* dotAlchemyDirectory;
// uv runs from the Worker root, so its arguments need absolute staging paths.
const staging = path.resolve(runtimeBase, dotAlchemy, "python", options.id);
const vendorDir = path.join(staging, "python_modules");
const tokenFile = path.join(staging, ".synced");
const token = yield* orDefault(fs.readFileString(tokenFile), undefined);
if (token === inputHash && (yield* orDefault(fs.exists(vendorDir), false))) {
return vendorDir;
}
yield* Effect.logDebug(
`Vendoring Python dependencies for ${pythonVersion} (emscripten-wasm32) with uv`,
);
yield* fs
.makeDirectory(staging, { recursive: true })
.pipe(Effect.mapError(uvError(`Failed to create "${staging}"`)));
// pywrangler always vendors the managed SDK package alongside the
// project's own dependencies; passing it as a second requirements input
// avoids mutating the user's pyproject.toml.
const extraRequirements = path.join(staging, "requirements.extra.txt");
yield* fs
.writeFileString(extraRequirements, `${MANAGED_SDK_PACKAGE}\n`)
.pipe(Effect.mapError(uvError(`Failed to write "${extraRequirements}"`)));
// 1. Resolve to a pylock.toml against the Pyodide wheel index. `--no-build`
// restricts the resolution to prebuilt (pure-Python or emscripten) wheels
// — building a Pyodide-platformed wheel locally is not possible.
const lockfile = path.join(staging, "pylock.toml");
yield* runUv(
[
"pip",
"compile",
...(hasPyproject ? [pyproject] : []),
extraRequirements,
"--python",
target.interpreter,
"--extra-index-url",
target.index,
"--index-strategy",
"unsafe-best-match",
"--no-header",
"--no-build",
"-o",
lockfile,
],
{ cwd: options.root },
);
// 2. Install the locked wheels into a fresh emscripten cross-venv.
const venv = path.join(staging, "venv");
yield* orDefault(fs.remove(venv, { recursive: true }), undefined);
yield* runUv(["venv", venv, "--python", target.interpreter], {
cwd: options.root,
});
yield* runUv(["pip", "install", "--no-build", "-r", lockfile, "--preview-features", "pylock"], {
cwd: options.root,
env: { VIRTUAL_ENV: venv },
});
// 3. The venv's site-packages IS the vendored python_modules directory.
const sitePackages =
process.platform === "win32"
? path.join(venv, "Lib", "site-packages")
: path.join(venv, "lib", `python${pythonVersion}`, "site-packages");
yield* orDefault(fs.remove(vendorDir, { recursive: true }), undefined);
yield* fs
.copy(sitePackages, vendorDir)
.pipe(Effect.mapError(uvError(`Failed to copy "${sitePackages}" to "${vendorDir}"`)));
// Mark the directory as a virtual environment (pywrangler parity).
yield* fs
.writeFileString(path.join(vendorDir, "pyvenv.cfg"), "")
.pipe(Effect.mapError(uvError("Failed to write pyvenv.cfg")));
yield* fs
.writeFileString(tokenFile, inputHash)
.pipe(Effect.mapError(uvError(`Failed to write "${tokenFile}"`)));
return vendorDir;
});
const orDefault = <A, B, R>(
effect: Effect.Effect<A, unknown, R>,
fallback: B,
): Effect.Effect<A | B, never, R> => Effect.catchCause(effect, () => Effect.succeed(fallback));
const globFiles = (
patterns: string[],
options: { cwd: string; ignore?: string[]; dot?: boolean },
) =>
Effect.tryPromise({
try: () =>
glob(patterns, {
cwd: options.cwd,
onlyFiles: true,
expandDirectories: false,
dot: options.dot ?? false,
ignore: options.ignore,
}),
catch: (error) =>
Bundle.BundleError.make({
message: `Failed to list Python worker files in "${options.cwd}"`,
cause: error,
}),
}).pipe(Effect.map((names) => names.map((name) => name.replaceAll("\\", "/")).sort()));
/**
* Read a Python Worker "bundle" from disk. There is no compilation step —
* the output is:
*
* 1. the entry module (first, so it becomes `main_module`), then every
* other `.py` file under the entry's directory, read as text;
* 2. every file of the resolved `python_modules/` directory (vendored
* wheels), read as bytes and named `python_modules/<relpath>` —
* Wrangler's vendored-module layout, which both the upload API and
* workerd resolve at runtime.
*/
export const readPythonWorkerBundle = Effect.fn(function* (options: PythonWorkerBundleOptions) {
const fs = yield* FileSystem.FileSystem;
const main = yield* resolveMainPath(options.main);
const root = path.dirname(main);
const runtimeBase = process.cwd();
const dotAlchemy = yield* dotAlchemyDirectory;
const entryName = path.basename(main);
const readTextModule = Effect.fn(function* (name: string) {
const content = yield* fs
.readFileString(path.join(root, name))
.pipe(
Effect.mapError(uvError(`Failed to read Python worker module "${path.join(root, name)}"`)),
);
const hash = yield* sha256(content);
return { path: name, content, hash } satisfies Bundle.BundleFile;
});
const sources = globFiles(["**/*.py"], {
cwd: root,
ignore: [
"**/__pycache__/**",
"python_modules/**",
"**/.venv*/**",
"**/node_modules/**",
"**/.alchemy/**",
`${convertPathToPattern(path.resolve(runtimeBase, dotAlchemy))}/**`,
],
}).pipe(
Effect.map((names) =>
names.filter(
(name) =>
name !== entryName && !isPathWithin(dotAlchemy, path.join(root, name), runtimeBase),
),
),
Effect.flatMap(Effect.forEach(readTextModule, { concurrency: "unbounded" })),
);
const vendored = resolvePythonModulesDir({ ...options, root }).pipe(
Effect.flatMap((vendorDir) => {
if (!vendorDir) {
return Effect.succeed([]);
}
const readVendoredModule = Effect.fn(function* (name: string) {
const content = yield* fs
.readFile(path.join(vendorDir, name))
.pipe(
Effect.mapError(
uvError(`Failed to read vendored module "${path.join(vendorDir, name)}"`),
),
);
const hash = yield* sha256(content);
return {
path: `python_modules/${name}`,
content,
hash,
} satisfies Bundle.BundleFile;
});
return globFiles(["**/*"], {
cwd: vendorDir,
dot: true,
// Compiled bytecode is host-specific and excluded by Wrangler too.
ignore: ["**/*.pyc", "**/__pycache__/**"],
}).pipe(Effect.flatMap(Effect.forEach(readVendoredModule, { concurrency: "unbounded" })));
}),
);
const [entry, additional, vendoredFiles] = yield* Effect.all(
[readTextModule(entryName), sources, vendored],
{ concurrency: "unbounded" },
);
return yield* Bundle.bundleOutputFromFiles([entry, ...additional, ...vendoredFiles]);
});
/**
* Watch a Python Worker for changes: emits the initial build, then
* rebuilds whenever a file under the entry's directory changes (debounced).
* Mirrors the {@link Bundle.BundleWatchEvent} protocol of the rolldown
* watcher so local dev can consume either stream interchangeably.
*/
export const watchPythonWorkerBundle = (options: PythonWorkerBundleOptions) =>
watchBundleDirectory({
main: options.main,
read: readPythonWorkerBundle(options),
ignore: (path) => path.includes("__pycache__") || path.includes("/python_modules/"),
});
/**
* Source provider for Python workers (`.py` entry): no bundling — the
* entry plus sibling `.py` files upload as Python modules with
* dependencies vendored via uv. The read is cached per run under the
* shared `"build"` artifact key so diff and reconcile vendor once.
*/
export const makePythonSource = (main: string): SourceProvider => {
const pythonOptions = (ctx: SourceContext) => ({
id: ctx.id,
fqn: ctx.fqn,
main,
compatibility: ctx.compatibility,
});
return bundleSource({
build: (ctx) => readPythonWorkerBundle(pythonOptions(ctx)).pipe(Artifacts.cached("build")),
watch: (ctx) => Effect.succeed(watchPythonWorkerBundle(pythonOptions(ctx))),
});
};