import { AbortMultipartUpload } from "@alchemy.run/aws/S3";
import { AbortMultipartUploadHttp } from "@alchemy.run/aws/S3";
import { CompleteMultipartUpload } from "@alchemy.run/aws/S3";
import { CompleteMultipartUploadHttp } from "@alchemy.run/aws/S3";
import { CreateMultipartUpload } from "@alchemy.run/aws/S3";
import { CreateMultipartUploadHttp } from "@alchemy.run/aws/S3";
import { DeleteObjects } from "@alchemy.run/aws/S3";
import { DeleteObjectsHttp } from "@alchemy.run/aws/S3";
import { GetObject } from "@alchemy.run/aws/S3";
import { GetObjectHttp } from "@alchemy.run/aws/S3";
import { HeadObject } from "@alchemy.run/aws/S3";
import { HeadObjectHttp } from "@alchemy.run/aws/S3";
import { ListObjectsV2 } from "@alchemy.run/aws/S3";
import { ListObjectsV2Http } from "@alchemy.run/aws/S3";
import { PutObject } from "@alchemy.run/aws/S3";
import { PutObjectHttp } from "@alchemy.run/aws/S3";
import { UploadPart } from "@alchemy.run/aws/S3";
import { UploadPartHttp } from "@alchemy.run/aws/S3";
/**
* `Git.BlobStoreS3` — AWS S3 implementation of {@link BlobStore}.
*
* Runs the git service's bulk bytes (packs, clone bundles, oversize
* objects, push spill) on an S3 bucket while compute stays on Cloudflare.
* It is built over the S3 bindings (`AWS.S3.GetObject`, `PutObject`, …),
* so one provision serves the Worker and the Repo DO alike, exactly like
* {@link BlobStoreR2}: the bucket is an `AWS.S3.Bucket` resource, the
* Worker gets a least-privilege IAM identity minted for it, and every
* request is signed with credentials assumed at runtime, in the bucket's
* region.
*
* Latency note: every blob operation crosses the internet to S3 (no
* Cloudflare-internal path), so serving-path reads are slower than R2.
* The window-coalescing in `ObjectStore`/`PackSource` keeps request
* counts low, but R2 remains the recommended default for
* Cloudflare-hosted assemblies.
*
* ### Providing the store
* **Example:** Packs on S3, compute on Cloudflare
* ```typescript
* const GitObjects = AWS.S3.Bucket("GitObjects");
*
* const GitLive = Git.ApiLive.pipe(
* Layer.provide(Git.ApiHandlersLive),
* Layer.provide(Git.ReposDurableObject),
* Layer.provide(Git.RegistryDurableObject),
* Layer.provide(Git.BlobStoreS3(GitObjects)),
* );
* ```
*/
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Stream from "effect/Stream";
import {
BlobStore,
BlobStoreError,
type BlobBody,
type BlobMeta,
orderedParts,
type BlobMultipart,
type BlobStoreShape,
} from "./BlobStore.ts";
const s3Error = (what: string) => (error: { readonly _tag?: string; readonly message?: string }) =>
new BlobStoreError({
reason: `${what}: ${error._tag ?? "S3Error"}${error.message ? `: ${error.message}` : ""}`,
});
const collectBytes = (
stream: Stream.Stream<Uint8Array, Error>,
what: string,
): Effect.Effect<Uint8Array, BlobStoreError> =>
Stream.runCollect(stream).pipe(
Effect.mapError((error) => s3Error(what)({ message: String(error) })),
Effect.map((chunks) => {
const total = chunks.reduce((n, c) => n + c.length, 0);
const out = new Uint8Array(total);
let at = 0;
for (const chunk of chunks) {
out.set(chunk, at);
at += chunk.length;
}
return out;
}),
);
/**
* S3-backed {@link BlobStore} using the given bucket. Accepts an
* `AWS.S3.Bucket` resource or its declaration Effect, like {@link BlobStoreR2}.
* See the module doc for identity and latency semantics.
*
* @layer
* @provides Git.BlobStore
*/
export const BlobStoreS3 = (bucket: Parameters<typeof GetObject>[0]): Layer.Layer<BlobStore> =>
Layer.effect(
BlobStore,
Effect.gen(function* () {
const getObject = yield* GetObject(bucket);
const putObject = yield* PutObject(bucket);
const headObject = yield* HeadObject(bucket);
const deleteObjects = yield* DeleteObjects(bucket);
const listObjectsV2 = yield* ListObjectsV2(bucket);
const createMultipart = yield* CreateMultipartUpload(bucket);
const uploadPart = yield* UploadPart(bucket);
const completeMultipart = yield* CompleteMultipartUpload(bucket);
const abortMultipart = yield* AbortMultipartUpload(bucket);
return {
get: (key, range) =>
getObject({
Key: key,
Range:
range === undefined
? undefined
: `bytes=${range.offset}-${range.offset + range.length - 1}`,
}).pipe(
Effect.map((output): BlobBody | null => {
const body = output.Body ?? Stream.empty;
return {
size: output.ContentLength ?? 0,
bytes: collectBytes(body, `get ${key}`),
stream: body.pipe(
Stream.mapError((error) => s3Error(`stream ${key}`)({ message: String(error) })),
),
};
}),
Effect.catchTag("NoSuchKey", () => Effect.succeed(null)),
Effect.mapError(s3Error(`get ${key}`)),
),
put: (key, body, opts) =>
putObject({
Key: key,
Body: body,
ContentLength: body instanceof Uint8Array ? body.length : opts?.contentLength,
}).pipe(Effect.mapError(s3Error(`put ${key}`)), Effect.asVoid),
head: (key) =>
headObject({ Key: key }).pipe(
Effect.map((output): BlobMeta => ({
key,
size: output.ContentLength ?? 0,
})),
Effect.catchTag("NotFound", () => Effect.succeed(null)),
Effect.mapError(s3Error(`head ${key}`)),
),
multipart: (key) =>
createMultipart({ Key: key }).pipe(
Effect.mapError(s3Error(`multipart ${key}`)),
Effect.map((created): BlobMultipart => {
const UploadId = created.UploadId ?? "";
return {
uploadId: UploadId,
uploadPart: (partNumber, part) =>
uploadPart({
Key: key,
UploadId,
PartNumber: partNumber,
Body: part,
ContentLength: part.length,
}).pipe(
Effect.mapError(s3Error(`part ${partNumber} of ${key}`)),
Effect.map((uploaded) => ({
partNumber,
etag: uploaded.ETag ?? "",
})),
),
complete: (parts) =>
completeMultipart({
Key: key,
UploadId,
MultipartUpload: {
Parts: orderedParts(parts).map((p) => ({
PartNumber: p.partNumber,
ETag: p.etag,
})),
},
}).pipe(Effect.mapError(s3Error(`complete ${key}`)), Effect.asVoid),
abort: abortMultipart({ Key: key, UploadId }).pipe(
Effect.mapError(s3Error(`abort ${key}`)),
Effect.asVoid,
),
};
}),
),
uploadPart: (key, uploadId, partNumber, part) =>
uploadPart({
Key: key,
UploadId: uploadId,
PartNumber: partNumber,
Body: part,
ContentLength: part.length,
}).pipe(
Effect.mapError(s3Error(`part ${partNumber} of ${key}`)),
Effect.map((uploaded) => ({
partNumber,
etag: uploaded.ETag ?? "",
})),
),
delete: (keys) => {
const list = typeof keys === "string" ? [keys] : [...keys];
if (list.length === 0) return Effect.void;
return deleteObjects({
Delete: {
Objects: list.map((Key) => ({ Key })),
Quiet: true,
},
}).pipe(Effect.mapError(s3Error("delete")), Effect.asVoid);
},
list: (prefix) =>
Stream.paginate(undefined as string | undefined, (ContinuationToken) =>
listObjectsV2({
Prefix: prefix,
ContinuationToken,
MaxKeys: 1000,
}).pipe(
Effect.mapError(s3Error(`list ${prefix}`)),
Effect.map((page) => {
const metas = (page.Contents ?? []).flatMap((object): Array<BlobMeta> =>
object.Key === undefined ? [] : [{ key: object.Key, size: object.Size ?? 0 }],
);
return [
metas,
page.IsTruncated && page.NextContinuationToken
? Option.some(page.NextContinuationToken)
: Option.none<string>(),
] as const;
}),
),
),
} satisfies BlobStoreShape;
}),
).pipe(
Layer.provide([
GetObjectHttp,
PutObjectHttp,
HeadObjectHttp,
DeleteObjectsHttp,
ListObjectsV2Http,
CreateMultipartUploadHttp,
UploadPartHttp,
CompleteMultipartUploadHttp,
AbortMultipartUploadHttp,
]),
) as never;