import * as rdc from "@distilled.cloud/cloudflare/r2-data-catalog";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Effect from "effect/Effect";
import * as Predicate from "effect/Predicate";
import * as Redacted from "effect/Redacted";
import * as Schedule from "effect/Schedule";
import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
import type { Providers } from "../Providers.ts";
import type { Bucket } from "./Bucket.ts";
const TypeId = "Cloudflare.R2.DataCatalog" as const;
type TypeId = typeof TypeId;
/**
* Whether a maintenance job (compaction or snapshot expiration) runs.
*/
export type MaintenanceState = "enabled" | "disabled";
/**
* Target output file size, in MB, that catalog compaction rewrites small
* files into.
*/
export type TargetSizeMb = "64" | "128" | "256" | "512";
/**
* Catalog-level compaction maintenance settings.
*/
export type Compaction = {
/**
* Whether compaction runs for tables in this catalog.
*/
state?: MaintenanceState;
/**
* Target output file size, in MB.
* @default "128"
*/
targetSizeMb?: TargetSizeMb;
};
/**
* Catalog-level snapshot expiration maintenance settings.
*/
export type SnapshotExpiration = {
/**
* Whether snapshot expiration runs for tables in this catalog.
*/
state?: MaintenanceState;
/**
* Maximum age a snapshot may reach before it is expired, expressed as a
* duration string (e.g. `"7d"`).
*/
maxSnapshotAge?: string;
/**
* Minimum number of snapshots retained per table regardless of age.
*/
minSnapshotsToKeep?: number;
};
export type DataCatalogProps = DataCatalogBucket & {
/**
* Compaction maintenance configuration. Only the fields you specify are
* enforced; omitted fields keep Cloudflare's defaults.
*/
compaction?: Compaction;
/**
* Snapshot expiration maintenance configuration. Only the fields you
* specify are enforced; omitted fields keep Cloudflare's defaults.
*/
snapshotExpiration?: SnapshotExpiration;
/**
* Cloudflare API token (with R2 read/write access) the catalog uses to
* run maintenance jobs against the bucket. Write-only: Cloudflare exposes
* only `credentialStatus: "present" | "absent"`, never the token itself.
* Maintenance jobs stay pending until a credential is provided.
*/
token?: Redacted.Redacted<string>;
};
/**
* The bucket a catalog is enabled on. `bucketName` is the deprecated spelling
* of `bucket`; both name the same bucket and switching between them is not a
* change.
*/
export type DataCatalogBucket =
| {
/**
* The R2 bucket to enable the Iceberg catalog on: a
* `Cloudflare.R2.Bucket` resource (orders the catalog after the bucket)
* or a bucket name. Changing the bucket replaces the catalog (the old
* bucket's catalog is disabled; table data is untouched).
*/
bucket: string | Bucket;
bucketName?: never;
}
| {
/**
* Name of the R2 bucket to enable the catalog on.
* @deprecated Use `bucket`, which also accepts the bucket resource.
*/
bucketName: string;
bucket?: never;
};
/**
* The bucket name behind a catalog's props: `bucket` as a name or a resolved
* `Cloudflare.R2.Bucket`, or the deprecated `bucketName`.
*/
export const catalogBucketName = (props: unknown): string | undefined => {
if (props === null || typeof props !== "object") return undefined;
const { bucket, bucketName } = props as { bucket?: unknown; bucketName?: unknown };
if (typeof bucket === "string") return bucket;
if (bucket !== null && typeof bucket === "object") {
const name = (bucket as { bucketName?: unknown }).bucketName;
if (typeof name === "string") return name;
}
return typeof bucketName === "string" ? bucketName : undefined;
};
export type DataCatalogAttributes = {
/**
* Unique identifier of the catalog (stable across disable/enable cycles).
*/
catalogId: string;
/**
* Catalog (warehouse) name, generated by Cloudflare as
* `{accountId}_{bucketName}`.
*/
name: string;
/**
* Name of the R2 bucket backing the catalog.
*/
bucketName: string;
/**
* The Cloudflare account the catalog belongs to.
*/
accountId: string;
/**
* Catalog status. A reconciled catalog is always `active`.
*/
status: "active" | "inactive";
/**
* Whether a maintenance credential is registered for this catalog.
*/
credentialStatus: "present" | "absent" | (string & {});
/**
* Observed compaction maintenance configuration.
*/
compaction:
| {
state: MaintenanceState;
targetSizeMb: TargetSizeMb;
}
| undefined;
/**
* Observed snapshot expiration maintenance configuration.
*/
snapshotExpiration:
| {
state: MaintenanceState;
maxSnapshotAge: string;
minSnapshotsToKeep: number;
}
| undefined;
/**
* Iceberg REST catalog URI for this warehouse — point PyIceberg, Spark, or
* any Iceberg REST client at this endpoint.
*/
catalogUri: string;
};
export type DataCatalog = Resource<
TypeId,
DataCatalogProps,
DataCatalogAttributes,
never,
Providers
>;
/**
* Apache Iceberg data catalog attached to a Cloudflare R2 bucket.
*
* R2 Data Catalog exposes an Iceberg REST catalog endpoint backed by an R2
* bucket, so engines like Spark, PyIceberg, and DuckDB can create and query
* Iceberg tables stored in R2. The catalog is a singleton per bucket: this
* resource enables it, keeps its maintenance configuration in sync, and
* disables it on destroy (table data in the bucket is never deleted).
*
* Also exported as `Cloudflare.Basin.Catalog` — the catalog of Cloudflare's
* Basin product (Pipelines + Catalog + SQL). The warehouse is named
* `<accountId>_<bucket>` and served at
* `https://catalog.cloudflarestorage.com/<accountId>/<bucket>`. Pass the
* catalog resource to a catalog `Cloudflare.Basin.Sink` (`catalog: Catalog`)
* so the sink is created after it.
* ### Enabling a catalog
* **Example:** Enable the catalog on an R2 bucket
* ```typescript
* const bucket = yield* Cloudflare.R2.Bucket("LakehouseBucket");
*
* const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", {
* bucket,
* });
*
* // Point any Iceberg REST client at the warehouse:
* const uri = catalog.catalogUri;
* const warehouse = catalog.name;
* ```
*
* ### Maintenance
* **Example:** Configure compaction and snapshot expiration
* ```typescript
* const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", {
* bucket,
* compaction: { state: "enabled", targetSizeMb: "256" },
* snapshotExpiration: {
* state: "enabled",
* maxSnapshotAge: "3d",
* minSnapshotsToKeep: 5,
* },
* });
* ```
*
* **Example:** Register a maintenance credential
* ```typescript
* // Maintenance jobs need an API token with R2 read/write on the bucket.
* const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", {
* bucket,
* compaction: { state: "enabled" },
* token: maintenanceToken, // Redacted<string>
* });
* ```
*
* @see https://developers.cloudflare.com/r2/data-catalog/
*
* @resource
* @product R2 Data Catalog
* @category Storage & Databases
*/
export const DataCatalog = Resource<DataCatalog>(TypeId);
/**
* Returns true if the given value is a DataCatalog (Basin Catalog) resource.
*/
export const isDataCatalog = (value: unknown): value is DataCatalog =>
Predicate.hasProperty(value, "Type") && value.Type === TypeId;
export const DataCatalogProvider = () =>
Provider.succeed(DataCatalog, {
stables: ["catalogId", "name", "accountId", "bucketName", "catalogUri"],
diff: Effect.fn(function* ({ olds, news, output }) {
const { accountId } = yield* yield* CloudflareEnvironment;
if (!isResolved(news)) return undefined;
if ((output?.accountId ?? accountId) !== accountId) {
return { action: "replace" } as const;
}
// The catalog is keyed to its bucket — moving buckets is a replacement
// (disable on the old bucket, enable on the new one). bucketName is an
// Input<string>; compare only once both sides are concrete strings.
const oldBucket = output?.bucketName ?? catalogBucketName(olds);
const newBucket = catalogBucketName(news);
if (oldBucket !== undefined && newBucket !== undefined && oldBucket !== newBucket) {
return { action: "replace" } as const;
}
return undefined;
}),
read: Effect.fn(function* ({ output, olds }) {
const { accountId } = yield* yield* CloudflareEnvironment;
const acct = output?.accountId ?? accountId;
const bucketName = output?.bucketName ?? catalogBucketName(olds);
if (bucketName === undefined) return undefined;
const observed = yield* getCatalog(acct, bucketName);
// `inactive` means the catalog was disabled — for reconciliation
// purposes the resource is gone (re-enable on the next deploy).
if (!observed || observed.status !== "active") return undefined;
const attrs = toAttributes(observed, acct);
// The catalog carries no ownership markers; if we have no state of our
// own, an active catalog may have been enabled out-of-band. Brand it
// `Unowned` so the engine gates takeover behind the adopt policy.
return output ? attrs : Unowned(attrs);
}),
list: Effect.fn(function* () {
const { accountId } = yield* yield* CloudflareEnvironment;
// R2 Data Catalog is an account-scoped collection: one warehouse per
// bucket that has the catalog enabled. Only `active` warehouses map to
// a live resource — `read` treats `inactive` (disabled) as gone.
return yield* rdc.listR2DataCatalogs({ accountId }).pipe(
Effect.map(({ warehouses }) =>
warehouses.filter((w) => w.status === "active").map((w) => toAttributes(w, accountId)),
),
// Accounts without R2 Data Catalog access reject the route entirely.
Effect.catchTag("InvalidRoute", () => Effect.succeed([])),
);
}),
reconcile: Effect.fn(function* ({ news, olds, output }) {
const { accountId } = yield* yield* CloudflareEnvironment;
const acct = output?.accountId ?? accountId;
// Inputs have been resolved by Plan: `bucket` is a name or the
// bucket's attributes.
const bucketName = catalogBucketName(news) as string;
// Observe — `output` is only a cache; a disabled or never-enabled
// catalog observes as missing/inactive and falls through to ensure.
let observed = yield* getCatalog(acct, bucketName);
// Ensure — enable is a true idempotent upsert (re-running it on an
// active catalog returns the same id), so no AlreadyExists race exists.
// A freshly-created bucket can briefly 404 the enable endpoint
// (`NoSuchBucket`, code 40406) — ride out that consistency lag.
if (!observed || observed.status !== "active") {
yield* rdc.enableR2DataCatalog({ accountId: acct, bucketName }).pipe(
Effect.retry({
while: (e) => e._tag === "NoSuchBucket",
schedule: catalogConsistencySchedule,
}),
);
observed = yield* rdc.getR2DataCatalog({ accountId: acct, bucketName }).pipe(
// The catalog can lag behind its own enable call.
Effect.retry({
while: (e) => e._tag === "WarehouseNotFound",
schedule: catalogConsistencySchedule,
}),
);
}
// Sync credential before maintenance — Cloudflare rejects maintenance
// updates when the warehouse has no credential registered.
// The API exposes only present/absent, so `olds` serves as the rotation
// hint: re-push when the token value changed or no credential is
// registered (adoption re-pushes; idempotent).
let credentialStatus = (observed.credentialStatus ?? "absent") as "present" | "absent";
if (news.token !== undefined) {
const rotated =
olds?.token === undefined || Redacted.value(olds.token) !== Redacted.value(news.token);
if (credentialStatus !== "present" || rotated) {
yield* rdc.createCredential({
accountId: acct,
bucketName,
token: Redacted.value(news.token),
});
credentialStatus = "present";
}
}
// Sync maintenance — diff observed config against the fields the user
// actually specified; skip the API entirely on a no-op.
let maintenance = observed.maintenanceConfig ?? undefined;
const observedCompaction = maintenance?.compaction ?? undefined;
const observedExpiration = maintenance?.snapshotExpiration ?? undefined;
const compactionDirty =
news.compaction !== undefined &&
((news.compaction.state !== undefined &&
news.compaction.state !== observedCompaction?.state) ||
(news.compaction.targetSizeMb !== undefined &&
news.compaction.targetSizeMb !== observedCompaction?.targetSizeMb));
const expirationDirty =
news.snapshotExpiration !== undefined &&
((news.snapshotExpiration.state !== undefined &&
news.snapshotExpiration.state !== observedExpiration?.state) ||
(news.snapshotExpiration.maxSnapshotAge !== undefined &&
news.snapshotExpiration.maxSnapshotAge !== observedExpiration?.maxSnapshotAge) ||
(news.snapshotExpiration.minSnapshotsToKeep !== undefined &&
news.snapshotExpiration.minSnapshotsToKeep !== observedExpiration?.minSnapshotsToKeep));
if (compactionDirty || expirationDirty) {
const updated = yield* rdc.updateMaintenanceConfig({
accountId: acct,
bucketName,
compaction: news.compaction,
snapshotExpiration: news.snapshotExpiration,
});
maintenance = {
compaction: updated.compaction,
snapshotExpiration: updated.snapshotExpiration,
};
}
return toAttributes(
{ ...observed, status: "active", maintenanceConfig: maintenance },
acct,
credentialStatus,
);
}),
delete: Effect.fn(function* ({ output }) {
// Disable is idempotent server-side; a never-enabled (or already
// disabled-and-bucket-deleted) catalog surfaces `WarehouseNotFound`.
yield* rdc
.disableR2DataCatalog({
accountId: output.accountId,
bucketName: output.bucketName,
})
.pipe(Effect.catchTag(["WarehouseNotFound", "NoSuchBucket"], () => Effect.void));
}),
});
// A freshly-created bucket (or freshly-enabled catalog) can briefly 404 on
// the r2-catalog endpoints before the warehouse propagates.
const catalogConsistencySchedule = Schedule.max([Schedule.exponential(100), Schedule.recurs(5)]);
/**
* Read a catalog by bucket name, mapping "gone" (`WarehouseNotFound`, code
* 40401, or a missing bucket) to `undefined`.
*/
const getCatalog = (accountId: string, bucketName: string) =>
rdc
.getR2DataCatalog({ accountId, bucketName })
.pipe(Effect.catchTag(["WarehouseNotFound", "NoSuchBucket"], () => Effect.undefined));
const toAttributes = (
catalog: rdc.GetR2DataCatalogResponse,
accountId: string,
credentialStatus?: "present" | "absent",
): DataCatalogAttributes => ({
catalogId: catalog.id,
name: catalog.name,
bucketName: catalog.bucket,
accountId,
// Distilled widens generated string enums to open unions (`string & {}`).
status: catalog.status as "active" | "inactive",
credentialStatus: credentialStatus ?? catalog.credentialStatus ?? "absent",
compaction: catalog.maintenanceConfig?.compaction
? {
state: catalog.maintenanceConfig.compaction.state as MaintenanceState,
targetSizeMb: catalog.maintenanceConfig.compaction.targetSizeMb as TargetSizeMb,
}
: undefined,
snapshotExpiration: catalog.maintenanceConfig?.snapshotExpiration
? {
state: catalog.maintenanceConfig.snapshotExpiration.state as MaintenanceState,
maxSnapshotAge: catalog.maintenanceConfig.snapshotExpiration.maxSnapshotAge,
minSnapshotsToKeep: catalog.maintenanceConfig.snapshotExpiration.minSnapshotsToKeep,
}
: undefined,
catalogUri: `https://catalog.cloudflarestorage.com/${accountId}/${catalog.bucket}`,
});