import * as bcm from "@distilled.cloud/aws/bcm-data-exports";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
/**
* The SQL query and table configurations for a data export.
*/
export interface ExportDataQuery {
/**
* The SQL statement selecting the columns to export, e.g.
* `SELECT identity_line_item_id, line_item_unblended_cost FROM COST_AND_USAGE_REPORT`.
* Data Exports supports a limited subset of SQL — see the AWS Data Exports
* table dictionary for available tables and columns.
*/
queryStatement: string;
/**
* Per-table property overrides, e.g.
* `{ COST_AND_USAGE_REPORT: { TIME_GRANULARITY: "HOURLY" } }`.
* Every table property has a default it assumes when omitted.
*/
tableConfigurations?: Record<string, Record<string, string>>;
}
/**
* Formatting options for the objects written to S3.
*/
export interface ExportS3OutputConfigurations {
/**
* The output type of the export.
* @default "CUSTOM"
*/
outputType?: "CUSTOM" | (string & {});
/**
* The file format of the exported objects.
* @default "TEXT_OR_CSV"
*/
format?: "TEXT_OR_CSV" | "PARQUET" | (string & {});
/**
* The compression of the exported objects. Use `PARQUET` compression with
* the `PARQUET` format.
* @default "GZIP"
*/
compression?: "GZIP" | "PARQUET" | (string & {});
/**
* Whether each delivery overwrites the previous report or writes a new one.
* @default "OVERWRITE_REPORT"
*/
overwrite?: "CREATE_NEW_REPORT" | "OVERWRITE_REPORT" | (string & {});
}
/**
* The S3 bucket the export is delivered to. The bucket policy must allow the
* `bcm-data-exports.amazonaws.com` and `billingreports.amazonaws.com` service
* principals to `s3:PutObject` and `s3:GetBucketPolicy`.
*/
export interface ExportS3Destination {
/**
* Name of the destination S3 bucket.
*/
s3Bucket: string;
/**
* Key prefix the export is written under.
*/
s3Prefix: string;
/**
* Region of the destination S3 bucket.
*/
s3Region: string;
/**
* Account ID that owns the destination bucket, for cross-account delivery.
*/
s3BucketOwner?: string;
/**
* Output formatting options.
* @default `{ outputType: "CUSTOM", format: "TEXT_OR_CSV", compression: "GZIP", overwrite: "OVERWRITE_REPORT" }`
*/
s3OutputConfigurations?: ExportS3OutputConfigurations;
}
export interface ExportProps {
/**
* Name of the export. Must be unique within the account. If omitted, a
* unique name is generated from the app, stage, and logical ID.
*
* Changing the name replaces the export.
*/
exportName?: string;
/**
* Description of the export.
*/
description?: string;
/**
* The SQL data query — the statement plus optional table configurations.
*/
dataQuery: ExportDataQuery;
/**
* The S3 destination the export is delivered to.
*/
s3Destination: ExportS3Destination;
/**
* How often the export refreshes. `SYNCHRONOUS` refreshes whenever the
* source billing data updates.
* @default `{ frequency: "SYNCHRONOUS" }`
*/
refreshCadence?: {
/**
* The refresh frequency.
* @default "SYNCHRONOUS"
*/
frequency: "SYNCHRONOUS" | (string & {});
};
/**
* Tags applied to the export.
*/
tags?: Record<string, string>;
}
export interface Export extends Resource<
"AWS.BCMDataExports.Export",
ExportProps,
{
/**
* Name of the export.
*/
exportName: string;
/**
* The ARN of the export.
*/
exportArn: string;
},
never,
Providers
> {}
/**
* An AWS Billing and Cost Management data export (Data Exports / CUR 2.0) —
* delivers billing and cost management data selected by an SQL query to an
* S3 bucket on a refresh cadence.
*
* Data Exports is a global service pinned to `us-east-1`; exports are free
* (standard S3 storage rates apply to the delivered objects).
*
* The destination bucket must grant the Data Exports service principals
* write access via its bucket policy (see the example below).
*
* ### Creating an Export
* **Example:** CUR 2.0 export to an S3 bucket
* ```typescript
* import * as AWS from "@alchemy.run/aws";
*
* const bucket = yield* AWS.S3.Bucket("BillingData", {
* forceDestroy: true,
* policy: [
* {
* Effect: "Allow",
* Principal: {
* Service: [
* "billingreports.amazonaws.com",
* "bcm-data-exports.amazonaws.com",
* ],
* },
* Action: ["s3:PutObject", "s3:GetBucketPolicy"],
* Resource: [bucket.bucketArn, AWS.interpolate`${bucket.bucketArn}/*`],
* },
* ],
* });
*
* const cur = yield* AWS.BCMDataExports.Export("Cur2", {
* dataQuery: {
* queryStatement:
* "SELECT identity_line_item_id, line_item_unblended_cost FROM COST_AND_USAGE_REPORT",
* tableConfigurations: {
* COST_AND_USAGE_REPORT: { TIME_GRANULARITY: "HOURLY" },
* },
* },
* s3Destination: {
* s3Bucket: bucket.bucketName,
* s3Prefix: "cur2",
* s3Region: "us-west-2",
* },
* });
* ```
*
* @resource
*/
export const Export = Resource<Export>("AWS.BCMDataExports.Export");
const retryThrottling = <A, E extends { readonly _tag: string }, R>(
self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
Effect.retry(self, {
while: (e) => e._tag === "ThrottlingException",
schedule: Schedule.max([Schedule.exponential("1 second"), Schedule.recurs(5)]),
});
export const ExportProvider = () =>
Provider.effect(
Export,
Effect.gen(function* () {
const createName = Effect.fn(function* (
id: string,
props: { exportName?: string | undefined },
) {
return props.exportName ?? (yield* createPhysicalName({ id, maxLength: 100 }));
});
const buildExport = (name: string, props: ExportProps): bcm.Export => ({
Name: name,
Description: props.description,
DataQuery: {
QueryStatement: props.dataQuery.queryStatement,
TableConfigurations: props.dataQuery.tableConfigurations,
},
DestinationConfigurations: {
S3Destination: {
S3Bucket: props.s3Destination.s3Bucket,
S3Prefix: props.s3Destination.s3Prefix,
S3Region: props.s3Destination.s3Region,
S3BucketOwner: props.s3Destination.s3BucketOwner,
S3OutputConfigurations: {
OutputType: props.s3Destination.s3OutputConfigurations?.outputType ?? "CUSTOM",
Format: props.s3Destination.s3OutputConfigurations?.format ?? "TEXT_OR_CSV",
Compression: props.s3Destination.s3OutputConfigurations?.compression ?? "GZIP",
Overwrite:
props.s3Destination.s3OutputConfigurations?.overwrite ?? "OVERWRITE_REPORT",
},
},
},
RefreshCadence: {
Frequency: props.refreshCadence?.frequency ?? "SYNCHRONOUS",
},
});
// Project both the observed and the desired definition through the
// same shape (fixed key order, defaults filled) so wire echoes — key
// ordering, an echoed S3BucketOwner — never register as drift.
const normalizeExport = (e: bcm.Export): string => {
const dest = e.DestinationConfigurations.S3Destination;
return JSON.stringify({
Name: e.Name,
Description: e.Description ?? "",
QueryStatement: e.DataQuery.QueryStatement,
TableConfigurations: Object.fromEntries(
Object.entries(e.DataQuery.TableConfigurations ?? {})
.sort(([l], [r]) => l.localeCompare(r))
.map(([table, props]) => [
table,
Object.fromEntries(
Object.entries(props ?? {}).sort(([l], [r]) => l.localeCompare(r)),
),
]),
),
S3Bucket: dest.S3Bucket,
S3Prefix: dest.S3Prefix,
S3Region: dest.S3Region,
OutputType: dest.S3OutputConfigurations.OutputType,
Format: dest.S3OutputConfigurations.Format,
Compression: dest.S3OutputConfigurations.Compression,
Overwrite: dest.S3OutputConfigurations.Overwrite,
Frequency: e.RefreshCadence.Frequency,
});
};
// S3BucketOwner is compared only when the user sets it — AWS may echo
// the effective owner account on reads even when it was never supplied.
const sameExport = (live: bcm.Export, desired: bcm.Export): boolean =>
normalizeExport(live) === normalizeExport(desired) &&
(desired.DestinationConfigurations.S3Destination.S3BucketOwner === undefined ||
desired.DestinationConfigurations.S3Destination.S3BucketOwner ===
live.DestinationConfigurations.S3Destination.S3BucketOwner);
const findArnByName = Effect.fn(function* (name: string) {
const matches = yield* bcm.listExports.items({}).pipe(
Stream.filter((e) => e.ExportName === name),
Stream.take(1),
Stream.runCollect,
);
return Array.from(matches)[0]?.ExportArn;
});
// Observe the live export definition. `arnHint` (from cached output)
// is verified, not trusted — a stale ARN falls through to name lookup.
const observe = Effect.fn(function* (name: string, arnHint: string | undefined) {
const arns = arnHint !== undefined ? [arnHint] : [];
const byName = yield* findArnByName(name);
if (byName !== undefined && byName !== arnHint) arns.push(byName);
for (const arn of arns) {
const found = yield* bcm
.getExport({ ExportArn: arn })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
if (found?.Export !== undefined) {
return { exportArn: arn, export: found.Export };
}
}
return undefined;
});
const fetchObservedTags = Effect.fn(function* (arn: string) {
return yield* bcm.listTagsForResource({ ResourceArn: arn }).pipe(
Effect.map((r) =>
Object.fromEntries((r.ResourceTags ?? []).map((t) => [t.Key, t.Value])),
),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed({} as Record<string, string>),
),
);
});
return Export.Provider.of({
stables: ["exportName", "exportArn"],
list: () =>
bcm.listExports.items({}).pipe(
Stream.runCollect,
Effect.map((chunk) =>
Array.from(chunk).map((e) => ({
exportName: e.ExportName,
exportArn: e.ExportArn,
})),
),
),
read: Effect.fn(function* ({ id, olds, output }) {
const name = output?.exportName ?? (yield* createName(id, olds ?? {}));
const live = yield* observe(name, output?.exportArn);
if (live === undefined) return undefined;
const attrs = { exportName: name, exportArn: live.exportArn };
const tags = yield* fetchObservedTags(live.exportArn);
return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
}),
// Replacement detection only — the export name is immutable; every
// other property updates in place via updateExport.
diff: Effect.fn(function* ({ id, news, olds }) {
if (!isResolved(news)) return undefined;
const oldName = yield* createName(id, olds);
const newName = yield* createName(id, news);
if (oldName !== newName) return { action: "replace" } as const;
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
const name = output?.exportName ?? (yield* createName(id, news));
const internalTags = yield* createInternalTags(id);
const desiredTags = { ...news.tags, ...internalTags };
const desired = buildExport(name, news);
// 1. OBSERVE — cloud state is authoritative; output only caches
// the ARN.
const live = yield* observe(name, output?.exportArn);
let exportArn: string;
if (live === undefined) {
// 2. ENSURE — create the export with all tags attached.
yield* session.note(`creating data export ${name}`);
const created = yield* retryThrottling(
bcm.createExport({
Export: desired,
ResourceTags: Object.entries(desiredTags).map(([Key, Value]) => ({ Key, Value })),
}),
);
exportArn = created.ExportArn ?? (yield* findArnByName(name)) ?? name;
} else {
exportArn = live.exportArn;
// 3. SYNC — updateExport overwrites the full definition; skip
// the call when the observed definition already matches.
if (!sameExport(live.export, desired)) {
yield* session.note(`updating data export ${name}`);
yield* retryThrottling(bcm.updateExport({ ExportArn: exportArn, Export: desired }));
}
}
// 3b. SYNC TAGS — diff against observed cloud tags.
const observedTags = yield* fetchObservedTags(exportArn);
const { upsert, removed } = diffTags(observedTags, desiredTags);
if (upsert.length > 0) {
yield* bcm.tagResource({
ResourceArn: exportArn,
ResourceTags: upsert,
});
}
if (removed.length > 0) {
yield* bcm.untagResource({
ResourceArn: exportArn,
ResourceTagKeys: removed,
});
}
yield* session.note(name);
return { exportName: name, exportArn };
}),
delete: Effect.fn(function* ({ output }) {
yield* bcm
.deleteExport({ ExportArn: output.exportArn })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
}),
});
}),
);