import * as securitylake from "@distilled.cloud/aws/securitylake";
import { Unowned } from "alchemy/AdoptPolicy";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import { toWireDays } from "alchemy/Util/Duration";
import * as Data from "effect/Data";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import { AWSEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import { readSecurityLakeTags, retryWhileConflict, toTagList } from "./internal.ts";
/**
* Encryption settings for a Security Lake Region.
*/
export interface DataLakeEncryptionConfiguration {
/**
* The KMS key ID used to encrypt objects at rest, or `S3_MANAGED_KEY` for
* SSE-S3 (the default).
* @default "S3_MANAGED_KEY"
*/
kmsKeyId?: string;
}
/**
* When objects expire out of the data lake.
*/
export interface DataLakeLifecycleExpiration {
/**
* How long after creation objects are deleted. Accepts any
* `Duration.Input` (e.g. `"365 days"`, `Duration.days(365)`; a bare
* number is milliseconds); the wire unit is whole days.
*/
days?: Duration.Input;
}
/**
* A storage-class transition rule.
*/
export interface DataLakeLifecycleTransition {
/** The S3 storage class to transition into (e.g. `GLACIER`, `ONEZONE_IA`). */
storageClass?: string;
/**
* How long after creation objects transition. Accepts any
* `Duration.Input` (e.g. `"30 days"`, `Duration.days(30)`; a bare
* number is milliseconds); the wire unit is whole days.
*/
days?: Duration.Input;
}
/**
* Lifecycle management (retention + storage-class transitions) for a Region.
*/
export interface DataLakeLifecycleConfiguration {
/** When objects expire out of the data lake. */
expiration?: DataLakeLifecycleExpiration;
/** Storage-class transition rules. */
transitions?: DataLakeLifecycleTransition[];
}
/**
* Rollup-Region replication for a contributing Region.
*/
export interface DataLakeReplicationConfiguration {
/** The rollup Regions that this Region's data is replicated into. */
regions?: string[];
/** IAM role ARN Security Lake assumes to replicate objects. */
roleArn?: string;
}
/**
* Per-Region configuration of the Security Lake data lake.
*/
export interface DataLakeRegionConfiguration {
/** The Region in which Security Lake is enabled. */
region: string;
/** Encryption at rest for this Region. */
encryptionConfiguration?: DataLakeEncryptionConfiguration;
/** Retention and storage-class transitions for this Region. */
lifecycleConfiguration?: DataLakeLifecycleConfiguration;
/** Rollup-Region replication for this Region. */
replicationConfiguration?: DataLakeReplicationConfiguration;
}
export interface DataLakeProps {
/**
* The Regions to enable Security Lake in, each with its own encryption,
* lifecycle, and replication settings. Adding a Region enables Security Lake
* there; removing a previously configured Region disables it there.
*/
configurations: DataLakeRegionConfiguration[];
/**
* ARN of the IAM role used by the Security Lake metastore manager (the
* `AmazonSecurityLakeMetaStoreManagerV2` role) to populate the Glue
* metastore with partition updates.
*/
metaStoreManagerRoleArn: string;
/**
* Tags applied to the data lake. Alchemy ownership tags are merged in
* automatically so the data lake can be recognized on subsequent runs.
*/
tags?: Record<string, string>;
}
/** Attributes of the data lake in one Region. */
export interface DataLakeRegionAttributes {
/** ARN of the data lake in this Region. */
dataLakeArn: string;
/** The Region. */
region: string;
/** ARN of the S3 bucket Security Lake created for this Region. */
s3BucketArn: string | undefined;
/** Creation status of the data lake in this Region. */
createStatus: string | undefined;
}
/** @resource */
export interface DataLake extends Resource<
"AWS.SecurityLake.DataLake",
DataLakeProps,
{
/** ARN of the data lake in the current (deployment) Region. */
dataLakeArn: string;
/** Every Region the data lake is enabled in. */
regions: string[];
/** Per-Region attributes (ARN, S3 bucket, status). */
dataLakes: DataLakeRegionAttributes[];
},
never,
Providers
> {}
/**
* The Amazon Security Lake data lake — the account-wide singleton that
* onboards the account to Security Lake. Enabling it creates S3 buckets,
* registers them with Lake Formation, and configures the Glue metastore in
* every configured Region.
*
* This is a heavyweight, account-wide resource: enabling/disabling Security
* Lake affects the whole account, and the S3 buckets it creates are retained
* after the data lake is deleted.
*
* ### Enabling Security Lake
* **Example:** Single-Region data lake
* ```typescript
* const lake = yield* SecurityLake.DataLake("Lake", {
* configurations: [{ region: "us-west-2" }],
* metaStoreManagerRoleArn: metastoreRole.roleArn,
* });
* ```
*
* **Example:** Lifecycle management and KMS encryption
* ```typescript
* const lake = yield* SecurityLake.DataLake("Lake", {
* configurations: [
* {
* region: "us-west-2",
* encryptionConfiguration: { kmsKeyId: key.keyId },
* lifecycleConfiguration: {
* expiration: { days: "365 days" },
* transitions: [{ storageClass: "ONEZONE_IA", days: "30 days" }],
* },
* },
* ],
* metaStoreManagerRoleArn: metastoreRole.roleArn,
* tags: { team: "security" },
* });
* ```
*/
const DataLakeResource = Resource<DataLake>("AWS.SecurityLake.DataLake");
export { DataLakeResource as DataLake };
/**
* The data lake failed to reach `COMPLETED` in one of the configured Regions.
*/
export class DataLakeCreateFailed extends Data.TaggedError("DataLakeCreateFailed")<{
readonly region: string;
readonly reason: string | undefined;
readonly code: string | undefined;
}> {}
// Convert a per-Region prop configuration (Duration-typed lifecycle days)
// into the wire shape the Security Lake API expects (whole days).
const toWireConfiguration = (
config: DataLakeRegionConfiguration,
): securitylake.DataLakeConfiguration => ({
region: config.region,
encryptionConfiguration: config.encryptionConfiguration,
lifecycleConfiguration: config.lifecycleConfiguration
? {
expiration: config.lifecycleConfiguration.expiration
? { days: toWireDays(config.lifecycleConfiguration.expiration.days) }
: undefined,
transitions: config.lifecycleConfiguration.transitions?.map((transition) => ({
storageClass: transition.storageClass,
days: toWireDays(transition.days),
})),
}
: undefined,
replicationConfiguration: config.replicationConfiguration,
});
// Normalize a per-Region config for observed↔desired comparison. AWS reports
// SSE-S3 as `S3_MANAGED_KEY`, so an unset kmsKeyId compares equal to it.
const normalizeConfig = (config: {
encryptionConfiguration?: securitylake.DataLakeEncryptionConfiguration;
lifecycleConfiguration?: securitylake.DataLakeLifecycleConfiguration;
replicationConfiguration?: securitylake.DataLakeReplicationConfiguration;
}) =>
JSON.stringify({
kmsKeyId: config.encryptionConfiguration?.kmsKeyId ?? "S3_MANAGED_KEY",
expirationDays: config.lifecycleConfiguration?.expiration?.days,
transitions: [...(config.lifecycleConfiguration?.transitions ?? [])]
.map((t) => ({ storageClass: t.storageClass, days: t.days }))
.sort((l, r) => (l.storageClass ?? "").localeCompare(r.storageClass ?? "")),
replicationRegions: [...(config.replicationConfiguration?.regions ?? [])].sort(),
replicationRoleArn: config.replicationConfiguration?.roleArn,
});
const buildRegionAttrs = (lake: securitylake.DataLakeResource): DataLakeRegionAttributes => ({
dataLakeArn: lake.dataLakeArn,
region: lake.region,
s3BucketArn: lake.s3BucketArn,
createStatus: lake.createStatus,
});
const buildAttrs = (lakes: securitylake.DataLakeResource[], currentRegion: string) => {
const sorted = [...lakes].sort((l, r) => l.region.localeCompare(r.region));
const current = sorted.find((lake) => lake.region === currentRegion);
return {
dataLakeArn: (current ?? sorted[0]!).dataLakeArn,
regions: sorted.map((lake) => lake.region),
dataLakes: sorted.map(buildRegionAttrs),
};
};
// All data lakes for the account across every Region.
const observeDataLakes = securitylake
.listDataLakes({})
.pipe(Effect.map((response) => [...(response.dataLakes ?? [])]));
// Observation for read/list: an account that never onboarded Security Lake
// rejects listDataLakes with AccessDeniedException — that means "no data
// lake", not a failure.
const observeDataLakesIfOnboarded = observeDataLakes.pipe(
Effect.catchTag(["AccessDeniedException", "UnauthorizedException"], () =>
Effect.succeed([] as securitylake.DataLakeResource[]),
),
);
export const DataLakeProvider = () =>
Provider.effect(
DataLakeResource,
Effect.gen(function* () {
// Bounded wait (~90s) for every desired Region to finish onboarding.
// A FAILED createStatus is surfaced as a typed error; still-PENDING
// Regions after the window are returned as-is (the next reconcile
// converges) rather than hanging.
const awaitDataLakeReady = Effect.fn(function* (regions: string[]) {
let lakes: securitylake.DataLakeResource[] = [];
for (let attempt = 0; attempt < 18; attempt++) {
lakes = yield* observeDataLakes;
const failed = lakes.find(
(lake) => regions.includes(lake.region) && lake.createStatus === "FAILED",
);
if (failed) {
return yield* new DataLakeCreateFailed({
region: failed.region,
reason: failed.updateStatus?.exception?.reason,
code: failed.updateStatus?.exception?.code,
});
}
const allReady = regions.every((region) =>
lakes.some((lake) => lake.region === region && lake.createStatus === "COMPLETED"),
);
if (allReady) return lakes;
yield* Effect.sleep("5 seconds");
}
return lakes;
});
return {
read: Effect.fn(function* ({ id }) {
const { region } = yield* AWSEnvironment.current;
const lakes = yield* observeDataLakesIfOnboarded;
if (lakes.length === 0) return undefined;
const attrs = buildAttrs(lakes, region);
const tags = yield* readSecurityLakeTags(attrs.dataLakeArn);
return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
}),
// The data lake is an account-wide singleton spanning Regions.
list: () =>
Effect.gen(function* () {
const { region } = yield* AWSEnvironment.current;
const lakes = yield* observeDataLakesIfOnboarded;
return lakes.length > 0 ? [buildAttrs(lakes, region)] : [];
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
const { region } = yield* AWSEnvironment.current;
const internalTags = yield* createInternalTags(id);
const desiredTags = { ...news.tags, ...internalTags };
const desiredConfigs = news.configurations.map(toWireConfiguration);
const desiredRegions = desiredConfigs.map((c) => c.region);
// 1. OBSERVE — cloud state is authoritative.
let lakes = yield* observeDataLakes;
const observedByRegion = new Map(lakes.map((lake) => [lake.region, lake]));
// 2. ENSURE — createDataLake is additive per Region, so enabling
// missing Regions covers both greenfield onboarding and Region
// expansion. A ConflictException is a race with a concurrent
// enable — fall through to observation.
const missing = desiredConfigs.filter((config) => !observedByRegion.has(config.region));
if (missing.length > 0) {
yield* securitylake
.createDataLake({
configurations: missing,
metaStoreManagerRoleArn: news.metaStoreManagerRoleArn,
tags: toTagList(desiredTags),
})
.pipe(Effect.catchTag("ConflictException", () => Effect.succeed({})));
}
// 3. SYNC per-Region settings — observed ↔ desired delta only.
const changed = desiredConfigs.filter((config) => {
const observed = observedByRegion.get(config.region);
return observed !== undefined && normalizeConfig(config) !== normalizeConfig(observed);
});
if (changed.length > 0) {
yield* securitylake.updateDataLake({
configurations: changed,
metaStoreManagerRoleArn: news.metaStoreManagerRoleArn,
});
}
// 3b. SYNC Region removals — only Regions this resource previously
// managed (recorded in output) are disabled; foreign Regions that
// happen to be enabled are left alone.
const staleRegions = (output?.regions ?? []).filter(
(managed) => !desiredRegions.includes(managed) && observedByRegion.has(managed),
);
if (staleRegions.length > 0) {
yield* securitylake.deleteDataLake({ regions: staleRegions }).pipe(retryWhileConflict);
}
// 3c. Wait (bounded) for onboarding to complete in every Region.
lakes = yield* awaitDataLakeReady(desiredRegions);
if (lakes.length === 0) {
return yield* new DataLakeCreateFailed({
region: desiredRegions[0] ?? region,
reason: "data lake not observable after create",
code: undefined,
});
}
const managed = lakes.filter((lake) => desiredRegions.includes(lake.region));
const attrs = buildAttrs(managed.length > 0 ? managed : lakes, region);
// 3d. SYNC tags — diff against OBSERVED cloud tags.
const observedTags = yield* readSecurityLakeTags(attrs.dataLakeArn);
const { upsert, removed } = diffTags(observedTags, desiredTags);
if (upsert.length > 0) {
yield* securitylake.tagResource({
resourceArn: attrs.dataLakeArn,
tags: upsert.map((t) => ({ key: t.Key, value: t.Value })),
});
}
if (removed.length > 0) {
yield* securitylake.untagResource({
resourceArn: attrs.dataLakeArn,
tagKeys: removed,
});
}
// 4. RETURN fresh attributes.
yield* session.note(attrs.dataLakeArn);
return attrs;
}),
// Disabling Security Lake retains the S3 buckets it created (AWS
// behavior); only log collection and the lake configuration go away.
delete: Effect.fn(function* ({ output }) {
if (output.regions.length === 0) return;
yield* securitylake.deleteDataLake({ regions: output.regions }).pipe(
retryWhileConflict,
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
);
}),
};
}),
);