import * as Cloudflare from "@alchemy.run/cloudflare";
import * as SQL from "@alchemy.run/cloudflare/D1/SQL";
import { sha256 } from "alchemy/Util/sha256";
import * as Clock from "effect/Clock";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import { Hex } from "effect/encoding";
import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
import * as Etag from "effect/http/Etag";
import * as FetchHttpClient from "effect/http/FetchHttpClient";
import * as HttpPlatform from "effect/http/HttpPlatform";
import * as HttpRouter from "effect/http/HttpRouter";
import { HttpServerRequest } from "effect/http/HttpServerRequest";
import * as HttpServerResponse from "effect/http/HttpServerResponse";
import * as Layer from "effect/Layer";
import * as Path from "effect/Path";
import * as Result from "effect/Result";
import * as Schema from "effect/Schema";
import { ManifestJson, type ManifestPackage } from "../Manifest.ts";
import {
BadRequest,
Forbidden,
manifestArtifactName,
MissingTarballs,
PackageTooLarge,
PkgApi,
RunNotInProgress,
RunRef,
Upstream,
} from "../Protocol.ts";
import { Bucket, Cache, COMMENT_MARKER, Database, RegistryConfig, tarballKey } from "./Bindings.ts";
import * as GitHub from "./GitHub.ts";
import * as KVCache from "./KVCache.ts";
import { sweep } from "./Sweep.ts";
import * as Tags from "./Tags.ts";
/** What GitHub says about a run. */
const Run = Schema.Struct({
...RunRef.fields,
headSha: Schema.String,
headBranch: Schema.NullOr(Schema.String),
headRepo: Schema.String,
pr: Schema.NullOr(Schema.Number),
});
type Run = typeof Run.Type;
const bindings = Layer.mergeAll(
Cloudflare.R2.ReadWriteBucketBinding,
Cloudflare.D1.QueryDatabaseBinding,
Cloudflare.KV.ReadWriteNamespaceBinding,
Cloudflare.Workers.CronEventSourceLive,
FetchHttpClient.layer,
);
const SHORT = 7;
/** How long a resolved run is reused for uploads before asking GitHub again. */
const RUN_CACHE = Duration.minutes(1);
const CHECK_NAME = "Preview packages";
/** The public origin a request arrived on. */
const origin = Effect.map(
HttpServerRequest,
(request) => `https://${request.headers.host ?? "localhost"}`,
);
/**
* A run from a fork: its head commit belongs to another repository, so its
* workflow file, and therefore everything it packs, is the fork's.
*/
const isFork = (run: Run) => run.headRepo !== run.repo;
/**
* Tags every package in a publication receives, all derived from the run.
* Runs on the repository's own commits get the head commit, the short
* commit, `branch:<name>`, and `pr:N` plus `pr:N:<short-sha>` for pull requests.
* A fork's run gets only `pr:N:<short-sha>`: commit tags are shared by every publisher, and a fork can
* run any commit it likes, including one already published from the
* repository, so letting it write them would let it repoint them.
*/
export const tagsFor = (run: Run): string[] => {
if (isFork(run)) {
return run.pr === null ? [] : [installTag(run)];
}
const tags = [run.headSha, run.headSha.slice(0, SHORT)];
if (run.pr !== null) tags.push(`pr:${run.pr}`, installTag(run));
if (run.headBranch) tags.push(`branch:${run.headBranch}`);
return tags;
};
/** The tag install commands are written against. */
export const installTag = (run: Run) =>
run.pr !== null ? `pr:${run.pr}:${run.headSha.slice(0, SHORT)}` : run.headSha.slice(0, SHORT);
const upstream = (e: { readonly _tag: string; readonly message?: string }) =>
Upstream.make({ message: GitHub.describe(e) });
/**
* Storage failures are not part of the protocol: they are defects, which
* the router answers with a 500 after logging the cause.
*/
const storageFailures = ["R2Error", "SqlError", "SchemaError"] as const;
/**
* Resolve the run a request names through GitHub; nothing about it is
* trusted from the client. Only allowed repositories are looked up at all,
* and the run must be in progress, since requests come from inside it.
*/
const lookupRun = Effect.fn("lookupRun")(function* (ref: RunRef) {
const { policy } = yield* RegistryConfig;
const github = yield* GitHub.GitHubApp;
if (!policy.repos.includes(ref.repo)) {
return yield* Forbidden.make({ message: `${ref.repo} may not publish` });
}
const data = yield* github.getRun(ref.repo, ref.runId).pipe(Effect.mapError(upstream));
if (
data.status !== "in_progress" ||
(data.run_attempt !== undefined && data.run_attempt !== ref.attempt)
) {
return yield* RunNotInProgress.make({
message: `run is not in progress: GitHub reports run ${ref.runId} as ${data.status}, attempt ${data.run_attempt ?? "unknown"}; the request is from attempt ${ref.attempt}`,
});
}
if (data.event !== "push" && data.event !== "pull_request") {
return yield* BadRequest.make({
message: `unsupported event ${data.event}`,
});
}
const headRepo = data.head_repository?.full_name ?? data.repository.full_name;
let pr: number | null = null;
if (data.event === "pull_request") {
if (data.head_branch === null) {
return yield* BadRequest.make({
message: `pull request run ${ref.runId} has no head branch`,
});
}
// The pull request whose head is this run's head, from the repository
// the run's head lives in. The same commit can head several pull
// requests, including one opened from a fork against a commit the
// repository already published; matching the head repository keeps a
// run's publication on its own pull request.
const pulls = yield* github
.pullRequestsForCommit(ref.repo, headRepo, data.head_branch, data.head_sha)
.pipe(Effect.mapError(upstream));
const first = pulls[0];
if (first === undefined) {
return yield* BadRequest.make({
message: `no open pull request from ${headRepo} has head ${data.head_sha}`,
});
}
pr = first.number;
} else if (headRepo !== ref.repo) {
return yield* BadRequest.make({
message: `push run ${ref.runId} is for ${headRepo}, not ${ref.repo}`,
});
}
return {
...ref,
headSha: data.head_sha,
headBranch: data.head_branch,
headRepo,
pr,
} satisfies Run;
});
/**
* The proof. The job uploaded the manifest as an artifact of its run,
* named by the manifest's hash. Only the job can add artifacts to the
* run, and GitHub reports the run's artifacts to the App, so a matching
* name means this run vouched for exactly this manifest text.
*/
const requireVouched = Effect.fn("requireVouched")(function* (run: Run, manifestText: string) {
const github = yield* GitHub.GitHubApp;
const expected = manifestArtifactName(yield* sha256(manifestText));
const artifacts = yield* github
.listRunArtifacts(run.repo, run.runId, expected)
.pipe(Effect.mapError(upstream));
if (!artifacts.some((a) => a.name === expected && !a.expired)) {
return yield* Forbidden.make({
message: `run ${run.runId} has not vouched for this manifest (no artifact ${expected})`,
});
}
});
const publish = Effect.fn("publish")(function* (run: Run, manifestText: string) {
const { policy } = yield* RegistryConfig;
const r2 = yield* Cloudflare.R2.ReadWriteBucket(Bucket);
const github = yield* GitHub.GitHubApp;
const base = yield* origin;
yield* requireVouched(run, manifestText);
const manifest = yield* Schema.decodeEffect(ManifestJson)(manifestText).pipe(
Effect.mapError((e) => BadRequest.make({ message: `invalid manifest: ${String(e)}` })),
);
// The run vouched for the manifest, but the tags come from the run's
// head, so the packed checkout has to be that commit: a `pull_request`
// job that packs the synthetic merge commit would otherwise publish bytes
// built from a commit that exists nowhere under the commit's name.
if (manifest.head !== run.headSha) {
return yield* BadRequest.make({
message: `manifest was packed at ${manifest.head}, but the run's head is ${run.headSha}`,
});
}
const packages = manifest.packages;
const maxSize = policy.maxPackageSize;
const tooLarge =
maxSize === undefined ? undefined : packages.find((pkg) => BigInt(pkg.size) > maxSize);
if (tooLarge !== undefined) {
return yield* PackageTooLarge.make({
message: `${tooLarge.name} exceeds ${maxSize} bytes`,
});
}
const missing = yield* Effect.filter(
packages,
(pkg) =>
r2.head(tarballKey(pkg.name, pkg.sha256)).pipe(Effect.map((object) => object === null)),
{ concurrency: 8 },
).pipe(Effect.map((packages) => packages.map(({ name, sha256 }) => ({ name, sha256 }))));
if (missing.length > 0) {
return yield* MissingTarballs.make({ missing });
}
const now = yield* Clock.currentTimeMillis;
const expiresAt = now + Duration.toMillis(policy.ttl);
const prs = run.pr !== null ? [`${run.repo}#${run.pr}`] : [];
const tags = tagsFor(run);
const tag = installTag(run);
const published = yield* Effect.forEach(
packages,
Effect.fn(function* (pkg: ManifestPackage) {
yield* Effect.forEach(
tags,
(tag) =>
Tags.upsert({
package: pkg.name,
tag,
sha256: pkg.sha256,
expiresAt,
prs,
}),
{ discard: true },
);
return {
name: pkg.name,
group: pkg.group,
url: `${base}/${pkg.name}/${tag}`,
tags,
};
}),
{ concurrency: 4 },
);
// A check on the commit itself, so the install lines are visible on
// pushes and on fork pull requests alike.
yield* github
.createCheckRun(run.repo, {
headSha: run.headSha,
name: CHECK_NAME,
title: `${packages.length} package(s) published`,
summary: GitHub.renderInstalls(published, manifest.groups),
detailsUrl: base,
})
.pipe(
Effect.catch((e) =>
Effect.logWarning(`check run on ${run.repo}@${run.headSha} failed: ${GitHub.describe(e)}`),
),
);
if (run.pr !== null) {
const body = GitHub.renderComment(published, manifest.groups, {
publishedAt: now,
expiresAt,
});
yield* github
.upsertComment(run.repo, run.pr, COMMENT_MARKER, body)
.pipe(
Effect.catch((e) =>
Effect.logWarning(`comment on ${run.repo}#${run.pr} failed: ${GitHub.describe(e)}`),
),
);
}
return { packages: published };
});
/**
* Content-addressed upload, streamed straight into R2 with the hash
* verified by R2 itself. Untagged objects are swept after a day.
*/
const uploadTarball = Effect.fn("uploadTarball")(function* (
request: HttpServerRequest,
name: string,
sha256: string,
) {
const { policy } = yield* RegistryConfig;
const r2 = yield* Cloudflare.R2.ReadWriteBucket(Bucket);
const contentLength = Number(request.headers["content-length"] ?? 0);
if (!Number.isSafeInteger(contentLength) || contentLength <= 0) {
return yield* BadRequest.make({ message: "Content-Length is required" });
}
const maxSize = policy.maxPackageSize;
if (maxSize !== undefined && BigInt(contentLength) > maxSize) {
return yield* PackageTooLarge.make({
message: `tarball exceeds ${maxSize} bytes`,
});
}
const key = tarballKey(name, sha256);
const existing = yield* r2.head(key);
if (existing !== null) {
return { name, sha256, size: existing.size, uploaded: false };
}
yield* r2
.put(key, request.stream, {
contentLength,
// The params schema already constrained this to 64 hex chars.
sha256: Result.getOrThrow(Hex.decode(sha256)),
})
.pipe(Effect.mapError((e) => BadRequest.make({ message: `upload rejected: ${String(e)}` })));
return { name, sha256, size: contentLength, uploaded: true };
});
/**
* Parse `/<name>/<tag>` and `/<name>/-/<sha256>.tgz`. Scoped names take
* two segments; the tag is everything after the name and may itself contain
* `/` and `:`.
*/
export const parseInstallPath = (
pathname: string,
scope: string | undefined,
):
| { kind: "tag"; name: string; tag: string }
| { kind: "tarball"; name: string; sha256: string }
| undefined => {
let path: string;
try {
path = decodeURIComponent(pathname.replace(/^\/+/, ""));
} catch {
// Malformed percent-encoding is a path nothing can be published under.
return undefined;
}
const tarball = path.match(/^(@[^/]+\/[^/@]+|[^/@]+)\/-\/([a-f0-9]{64})\.tgz$/);
if (tarball) {
return {
kind: "tarball",
name: qualify(tarball[1]!, scope),
sha256: tarball[2]!,
};
}
const segments = path.split("/");
const nameLength = path.startsWith("@") ? 2 : 1;
const name = segments.slice(0, nameLength).join("/");
const tag = segments.slice(nameLength).join("/");
if (segments.length <= nameLength || name === "" || tag === "") {
return undefined;
}
return { kind: "tag", name: qualify(name, scope), tag };
};
const qualify = (name: string, scope: string | undefined) =>
scope !== undefined && !name.startsWith("@") ? `${scope}/${name}` : name;
const notFound = (message: string) => HttpServerResponse.json({ error: message }, { status: 404 });
/**
* Install URLs: `/<name>/<tag>` redirects to the immutable tarball URL,
* `/<name>/-/<sha256>.tgz` streams it from R2. Everything the API routes
* do not claim lands here.
*/
const install = Effect.gen(function* () {
const request = yield* HttpServerRequest;
const { aliases } = yield* RegistryConfig;
const r2 = yield* Cloudflare.R2.ReadWriteBucket(Bucket);
if (request.method !== "GET" && request.method !== "HEAD") {
return yield* HttpServerResponse.json({ error: "method not allowed" }, { status: 405 });
}
const host = request.headers.host ?? "localhost";
const target = parseInstallPath(new URL(request.url, `https://${host}`).pathname, aliases[host]);
if (target === undefined) {
return yield* notFound("not found");
}
if (target.kind === "tag") {
const sha256 = yield* Tags.resolve(target.name, target.tag);
if (sha256 === undefined) {
return yield* notFound(`${target.name}@${target.tag} not found`);
}
return HttpServerResponse.redirect(`/${target.name}/-/${sha256}.tgz`, {
status: 302,
headers: { "cache-control": "no-store" },
});
}
const object = yield* r2.get(tarballKey(target.name, target.sha256));
if (object === null) {
return yield* notFound("tarball not found");
}
const headers = {
"content-type": "application/gzip",
"content-length": String(object.size),
"cache-control": "public, max-age=31536000, immutable",
};
return request.method === "HEAD"
? HttpServerResponse.empty({ status: 200, headers })
: HttpServerResponse.stream(object.body, { status: 200, headers });
});
const banner = Effect.map(origin, (base) =>
HttpServerResponse.text(
`Preview package registry. Install with: bun add ${base}/<package>/<commit|branch:name|pr:N|pr:N:short-sha>\n`,
),
);
/**
* The registry never serves files, so the HTTP platform's file surface is
* stubbed rather than pulling a file system into the Worker.
*/
const HttpPlatformStub = Layer.succeed(HttpPlatform.HttpPlatform, {
platform: "web",
compression: {
algorithms: new Set<HttpPlatform.CompressionAlgorithm>(),
compressResponse: (response) => Effect.succeed(response),
},
fileResponse: () => Effect.die("fileResponse is not supported"),
fileWebResponse: () => Effect.die("fileWebResponse is not supported"),
});
/**
* The registry Worker's Init phase. Runs at plan time to register bindings
* and the cron trigger, and once per isolate at runtime to build the
* router. Everything it needs comes from {@link RegistryConfig}.
*/
export const handler = Effect.gen(function* () {
const config = yield* RegistryConfig;
// Binding the capabilities here is what registers the R2, D1, and KV
// bindings on the Worker at plan time. Handlers bind them again per event,
// which at runtime is just a lookup in the Worker's environment.
yield* Cloudflare.R2.ReadWriteBucket(Bucket);
yield* Cloudflare.KV.ReadWriteNamespace(Cache);
const d1 = yield* Cloudflare.D1.QueryDatabase(Database);
// Built once per isolate. The D1 client memoizes its connection on each
// event's scope, so its layer is safe to share.
const github = yield* GitHub.GitHubApp.pipe(Effect.provide(GitHub.GitHubAppLive));
const services = Layer.mergeAll(
Cloudflare.R2.ReadWriteBucketBinding,
Cloudflare.KV.ReadWriteNamespaceBinding,
SQL.D1Layer(d1),
Layer.succeed(GitHub.GitHubApp, github),
Layer.succeed(RegistryConfig, config),
);
// Recently resolved runs are reused so the uploads that follow a publish
// do not repeat the lookup. Failures are not kept.
const runs = KVCache.make(
(ref: RunRef) => `run:${ref.repo}#${ref.runId}:${ref.attempt}`,
Run,
(ref) => Effect.map(lookupRun(ref), (value) => ({ value, ttl: RUN_CACHE })),
);
yield* Cloudflare.Workers.cron(config.cron, () => sweep.pipe(Effect.provide(services)));
const registry = HttpApiBuilder.group(PkgApi, "Registry", (handlers) =>
handlers
.handle("publish", ({ payload }) =>
Effect.flatMap(runs(payload.run), (run) => publish(run, payload.manifest)).pipe(
Effect.catchTag(storageFailures, (e) => Effect.die(e)),
),
)
// Raw so the body streams into R2 instead of being buffered.
.handleRaw("uploadTarball", ({ query, params, request }) =>
Effect.andThen(runs(query), uploadTarball(request, params.name, params.sha256)).pipe(
Effect.catchTag("R2Error", (e) => Effect.die(e)),
),
),
);
const healthGroup = HttpApiBuilder.group(PkgApi, "Health", (handlers) =>
handlers.handle("health", () => Effect.succeed({ ok: true })),
);
// Unlike `HttpApiBuilder.group`, the router defers a route's requirements
// to request time rather than the layer build, so the services are
// provided on the route effect itself.
const installRoutes = HttpRouter.use((router) =>
Effect.all([
router.add("GET", "/", banner),
router.add(
"*",
"/*",
install.pipe(
Effect.catchTag(storageFailures, (e) => Effect.die(e)),
Effect.provide(services),
),
),
]),
);
return {
fetch: Layer.mergeAll(HttpApiBuilder.layer(PkgApi), installRoutes).pipe(
Layer.provide([registry, healthGroup]),
Layer.provide([Etag.layer, HttpPlatformStub, Path.layer]),
Layer.provide(services),
HttpRouter.toHttpEffect,
),
};
}).pipe(Effect.provide(bindings));