import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
import * as mwaa from "@distilled.cloud/aws/mwaa-serverless";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
export interface WorkflowDefinitionS3Location {
/**
* Name of the S3 bucket that holds the workflow definition YAML file.
*/
bucket: string;
/**
* Key of the workflow definition YAML object within the bucket.
*/
objectKey: string;
/**
* Specific S3 object version of the definition to use. Omit to use the
* latest version.
*/
versionId?: string;
}
export interface WorkflowEncryptionConfiguration {
/**
* How workflow data is encrypted — `"AWS_MANAGED_KEY"` (the default) or
* `"CUSTOMER_MANAGED_KEY"`.
*
* Encryption cannot be changed in place — changing it replaces the
* workflow.
* @default "AWS_MANAGED_KEY"
*/
type: mwaa.EncryptionType;
/**
* ID or ARN of the customer managed KMS key to encrypt workflow data
* with. Required when `type` is `"CUSTOMER_MANAGED_KEY"`.
*/
kmsKeyId?: string;
}
export interface WorkflowLoggingConfiguration {
/**
* Name of the CloudWatch log group that receives task logs for the
* workflow's runs.
*/
logGroupName: string;
}
export interface WorkflowNetworkConfiguration {
/**
* Security group IDs applied to tasks that access resources inside your
* VPC.
*/
securityGroupIds?: string[];
/**
* Subnet IDs the workflow's tasks run in when accessing resources inside
* your VPC.
*/
subnetIds?: string[];
}
export interface WorkflowProps {
/**
* Name of the workflow. If omitted, a deterministic physical name is
* generated from the app, stage, and logical ID.
*
* Changing the name replaces the workflow.
* @default ${app}-${stage}-${id}
*/
name?: string;
/**
* S3 location of the workflow definition — a YAML file that defines the
* DAG structure using supported AWS operators.
*
* Updating the definition creates a new workflow version and disables
* scheduling on all previous versions.
*/
definitionS3Location: WorkflowDefinitionS3Location;
/**
* ARN of the IAM execution role that Amazon MWAA Serverless assumes to
* run the workflow's tasks. The role's trust policy must allow the
* `airflow-serverless.amazonaws.com` service principal to assume it.
*/
roleArn: string;
/**
* Human-readable description of the workflow.
*/
description?: string;
/**
* Encryption configuration for workflow data. Omit to use an AWS managed
* key. Changing encryption replaces the workflow (it cannot be updated
* in place).
* @default AWS managed key
*/
encryptionConfiguration?: WorkflowEncryptionConfiguration;
/**
* CloudWatch logging configuration for the workflow's task logs.
*/
loggingConfiguration?: WorkflowLoggingConfiguration;
/**
* Version of the workflow definition engine.
* @default 1
*/
engineVersion?: mwaa.EngineVersion;
/**
* Network configuration for tasks that access resources inside your VPC.
*/
networkConfiguration?: WorkflowNetworkConfiguration;
/**
* How workflow runs are triggered (for example on a schedule defined in
* the workflow definition, or only on demand).
*/
triggerMode?: string;
/**
* Tags to apply to the workflow. Merged with internal Alchemy tags.
*/
tags?: Record<string, string>;
}
export interface Workflow extends Resource<
"AWS.MWAAServerless.Workflow",
WorkflowProps,
{
/** Name of the workflow. */
name: string;
/** ARN of the workflow. */
workflowArn: string;
/** Latest workflow version (a new version is published on each update). */
workflowVersion: string | undefined;
/** Current lifecycle status of the workflow. */
workflowStatus: mwaa.WorkflowStatus | undefined;
/** ARN of the IAM role the workflow's tasks assume. */
roleArn: string | undefined;
/** How runs are triggered (e.g. scheduled or on-demand). */
triggerMode: string | undefined;
},
never,
Providers
> {}
/**
* An Amazon Managed Workflows for Apache Airflow **Serverless** workflow —
* a serverless Airflow DAG defined by a YAML file in S3 and executed by
* AWS-managed, multi-tenant Airflow infrastructure without provisioning an
* environment.
*
* Each update to the definition or configuration creates a new workflow
* version; MWAA Serverless keeps only the latest version actively
* scheduled.
* ### Creating a Workflow
* **Example:** Basic Workflow
* ```typescript
* import * as MWAAServerless from "@alchemy.run/aws/MWAAServerless";
* import * as IAM from "@alchemy.run/aws/IAM";
*
* const role = yield* IAM.Role("WorkflowRole", {
* assumeRolePolicyDocument: {
* Version: "2012-10-17",
* Statement: [{
* Effect: "Allow",
* Principal: { Service: "airflow-serverless.amazonaws.com" },
* Action: ["sts:AssumeRole"],
* }],
* },
* });
*
* const workflow = yield* MWAAServerless.Workflow("Etl", {
* definitionS3Location: {
* bucket: "my-dag-bucket",
* objectKey: "workflows/etl.yaml",
* },
* roleArn: role.roleArn,
* });
* ```
*
* **Example:** Workflow with Logging and Tags
* ```typescript
* const workflow = yield* MWAAServerless.Workflow("Etl", {
* definitionS3Location: {
* bucket: "my-dag-bucket",
* objectKey: "workflows/etl.yaml",
* },
* roleArn: role.roleArn,
* description: "nightly ETL",
* loggingConfiguration: { logGroupName: "/mwaa-serverless/etl" },
* tags: { team: "data" },
* });
* ```
*
* @resource
*/
export const Workflow = Resource<Workflow>("AWS.MWAAServerless.Workflow");
/**
* Bounded retry for `createWorkflow` while a freshly created IAM execution
* role propagates — the service validates that it can assume the role at
* create time, which surfaces as a ValidationException/AccessDeniedException
* mentioning the role for the first seconds of the role's life.
*
* Explicitly annotated so the conditional `Retry.Return` type never leaks
* into declaration emit (it would widen `AWS.providers()` for consumers).
*/
const retryWhileRolePropagates = <A, R>(
self: Effect.Effect<A, mwaa.CreateWorkflowError, R>,
): Effect.Effect<A, mwaa.CreateWorkflowError, R> =>
Effect.retry(self, {
while: (e) =>
(e._tag === "ValidationException" || e._tag === "AccessDeniedException") &&
/role/i.test(e.message ?? ""),
schedule: Schedule.max([Schedule.fixed("3 seconds"), Schedule.recurs(8)]),
});
export const WorkflowProvider = () =>
Provider.effect(
Workflow,
Effect.gen(function* () {
const createName = Effect.fn(function* (id: string, props: Pick<WorkflowProps, "name">) {
return props.name ?? (yield* createPhysicalName({ id, maxLength: 64 }));
});
const observeByArn = (arn: string) =>
mwaa
.getWorkflow({ WorkflowArn: arn })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
const findArnByName = (name: string) =>
mwaa.listWorkflows.items({}).pipe(
Stream.runCollect,
Effect.map((items) => Array.from(items).find((w) => w.Name === name)?.WorkflowArn),
);
const observe = Effect.fn(function* (name: string, arnHint: string | undefined) {
if (arnHint !== undefined) {
const found = yield* observeByArn(arnHint);
if (found !== undefined) return found;
}
const arn = yield* findArnByName(name);
return arn === undefined ? undefined : yield* observeByArn(arn);
});
const observedTags = (arn: string) =>
mwaa.listTagsForResource({ ResourceArn: arn }).pipe(
Effect.map((r) => {
const tags: Record<string, string> = {};
for (const [key, value] of Object.entries(r.Tags ?? {})) {
if (value !== undefined) tags[key] = value;
}
return tags;
}),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed({} as Record<string, string>),
),
);
const toDefinitionS3Location = (
location: WorkflowDefinitionS3Location,
): mwaa.DefinitionS3Location => ({
Bucket: location.bucket,
ObjectKey: location.objectKey,
VersionId: location.versionId,
});
const sameDefinition = (
live: mwaa.DefinitionS3Location | undefined,
desired: WorkflowDefinitionS3Location,
) =>
live !== undefined &&
live.Bucket === desired.bucket &&
live.ObjectKey === desired.objectKey &&
(desired.versionId === undefined || live.VersionId === desired.versionId);
const sameNetwork = (
live: mwaa.NetworkConfiguration | undefined,
desired: WorkflowNetworkConfiguration,
) =>
JSON.stringify({
securityGroupIds: live?.SecurityGroupIds ?? [],
subnetIds: live?.SubnetIds ?? [],
}) ===
JSON.stringify({
securityGroupIds: desired.securityGroupIds ?? [],
subnetIds: desired.subnetIds ?? [],
});
const toAttributes = (live: mwaa.GetWorkflowResponse, name: string) => ({
name: live.Name ?? name,
workflowArn: live.WorkflowArn,
workflowVersion: live.WorkflowVersion,
workflowStatus: live.WorkflowStatus,
roleArn: live.RoleArn,
triggerMode: live.TriggerMode,
});
return Workflow.Provider.of({
stables: ["name", "workflowArn"],
list: () =>
Effect.gen(function* () {
const items = yield* mwaa.listWorkflows.items({}).pipe(Stream.runCollect);
const workflows: {
name: string;
workflowArn: string;
workflowVersion: string | undefined;
workflowStatus: mwaa.WorkflowStatus | undefined;
roleArn: string | undefined;
triggerMode: string | undefined;
}[] = [];
for (const item of Array.from(items)) {
if (item.WorkflowArn === undefined || item.Name === undefined) {
continue;
}
// tolerate delete races between list and get
const live = yield* observeByArn(item.WorkflowArn);
if (live !== undefined && live.Name !== undefined) {
workflows.push(toAttributes(live, live.Name));
}
}
return workflows;
}),
read: Effect.fn(function* ({ id, olds, output }) {
const name = output?.name ?? (yield* createName(id, olds ?? {}));
const live = yield* observe(name, output?.workflowArn);
if (live === undefined || live.Name === undefined) {
return undefined;
}
const attrs = toAttributes(live, name);
const tags = yield* observedTags(live.WorkflowArn);
return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
}),
diff: Effect.fn(function* ({ id, news, olds }) {
if (!isResolved(news)) return undefined;
const oldName = yield* createName(id, olds ?? {});
const newName = yield* createName(id, news ?? {});
if (oldName !== newName) {
return { action: "replace" } as const;
}
// Encryption cannot be updated in place — UpdateWorkflow does not
// accept EncryptionConfiguration.
const oldEncryption = olds?.encryptionConfiguration;
const newEncryption = news.encryptionConfiguration;
if (
(oldEncryption?.type ?? "AWS_MANAGED_KEY") !==
(newEncryption?.type ?? "AWS_MANAGED_KEY") ||
oldEncryption?.kmsKeyId !== newEncryption?.kmsKeyId
) {
return { action: "replace" } as const;
}
// fall through: engine default update logic for mutable fields
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
const name = output?.name ?? (yield* createName(id, news));
const internalTags = yield* createInternalTags(id);
const desiredTags = { ...news.tags, ...internalTags };
// 1. OBSERVE — cloud state is authoritative; output.workflowArn is
// only a cache of the identifier.
let live = yield* observe(name, output?.workflowArn);
// 2. ENSURE — create when missing. A ConflictException means a
// concurrent create (or a previous run's create whose state
// persistence failed) raced us: converge on the workflow with
// OUR name if one now exists, otherwise propagate.
if (live === undefined) {
live = yield* mwaa
.createWorkflow({
Name: name,
DefinitionS3Location: toDefinitionS3Location(news.definitionS3Location),
RoleArn: news.roleArn,
Description: news.description,
EncryptionConfiguration:
news.encryptionConfiguration === undefined
? undefined
: {
Type: news.encryptionConfiguration.type,
KmsKeyId: news.encryptionConfiguration.kmsKeyId,
},
LoggingConfiguration:
news.loggingConfiguration === undefined
? undefined
: {
LogGroupName: news.loggingConfiguration.logGroupName,
},
EngineVersion: news.engineVersion,
NetworkConfiguration:
news.networkConfiguration === undefined
? undefined
: {
SecurityGroupIds: news.networkConfiguration.securityGroupIds,
SubnetIds: news.networkConfiguration.subnetIds,
},
TriggerMode: news.triggerMode,
Tags: desiredTags,
})
.pipe(
retryWhileRolePropagates,
Effect.flatMap((created) => observeByArn(created.WorkflowArn)),
Effect.catchTag("ConflictException", (error) =>
observe(name, undefined).pipe(
Effect.flatMap((existing) =>
existing === undefined ? Effect.fail(error) : Effect.succeed(existing),
),
),
),
);
}
const arn = live?.WorkflowArn ?? output?.workflowArn;
// 3. SYNC — diff each OBSERVED mutable aspect against the desired
// state; UpdateWorkflow requires the definition and role, so
// issue one full update when any managed aspect drifted (each
// update creates a new workflow version — skip on no-op).
if (live !== undefined && arn !== undefined) {
const drifted =
!sameDefinition(live.DefinitionS3Location, news.definitionS3Location) ||
live.RoleArn !== news.roleArn ||
(news.description !== undefined && live.Description !== news.description) ||
(news.loggingConfiguration !== undefined &&
live.LoggingConfiguration?.LogGroupName !==
news.loggingConfiguration.logGroupName) ||
(news.engineVersion !== undefined && live.EngineVersion !== news.engineVersion) ||
(news.networkConfiguration !== undefined &&
!sameNetwork(live.NetworkConfiguration, news.networkConfiguration)) ||
(news.triggerMode !== undefined && live.TriggerMode !== news.triggerMode);
if (drifted) {
yield* mwaa.updateWorkflow({
WorkflowArn: arn,
DefinitionS3Location: toDefinitionS3Location(news.definitionS3Location),
RoleArn: news.roleArn,
Description: news.description,
LoggingConfiguration:
news.loggingConfiguration === undefined
? undefined
: {
LogGroupName: news.loggingConfiguration.logGroupName,
},
EngineVersion: news.engineVersion,
NetworkConfiguration:
news.networkConfiguration === undefined
? undefined
: {
SecurityGroupIds: news.networkConfiguration.securityGroupIds,
SubnetIds: news.networkConfiguration.subnetIds,
},
TriggerMode: news.triggerMode,
});
live = (yield* observeByArn(arn)) ?? live;
}
}
// 3b. SYNC TAGS — diff against OBSERVED cloud tags so adoption
// converges (create-time tags only apply on first create).
if (arn !== undefined) {
const currentTags = yield* observedTags(arn);
const { upsert, removed } = diffTags(currentTags, desiredTags);
if (upsert.length > 0) {
yield* mwaa.tagResource({
ResourceArn: arn,
Tags: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
});
}
if (removed.length > 0) {
yield* mwaa.untagResource({
ResourceArn: arn,
TagKeys: removed,
});
}
}
yield* session.note(name);
return {
name: live?.Name ?? name,
workflowArn: arn!,
workflowVersion: live?.WorkflowVersion,
workflowStatus: live?.WorkflowStatus,
roleArn: live?.RoleArn ?? news.roleArn,
triggerMode: live?.TriggerMode ?? news.triggerMode,
};
}),
delete: Effect.fn(function* ({ output }) {
// Omitting WorkflowVersion deletes the workflow and all versions.
yield* mwaa
.deleteWorkflow({ WorkflowArn: output.workflowArn })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
// MWAA Serverless auto-creates a per-workflow log group named
// `/aws/mwaa-serverless/{name}-{id}/` (the ARN's resource id,
// which includes the service-assigned 10-char suffix, plus a
// trailing slash) at CREATE time, and deleteWorkflow does NOT
// remove it — without this reap every deleted workflow (including
// the old workflow of a replacement) leaks an orphaned log group.
// A group with no streams or provably-quiescent ingestion (last
// ingestion > 2 minutes ago) has no pending flush and deletes in
// a single call, so routine deletes of never-run workflows stay
// fast. A group with recent ingestion — or one that does not
// exist yet — is re-reaped on a short bounded schedule
// (t=0s / 20s / 40s) to catch a late log flush from the
// workflow's final runs; each attempt is idempotent.
const workflowId = output.workflowArn.split(":workflow/")[1];
if (workflowId !== undefined) {
const logGroupName = `/aws/mwaa-serverless/${workflowId}/`;
const reapLogGroup = logs
.deleteLogGroup({ logGroupName })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
const observed = yield* logs
.describeLogStreams({
logGroupName,
orderBy: "LastEventTime",
descending: true,
limit: 1,
})
.pipe(
Effect.map((r) => ({
exists: true,
lastIngestion: r.logStreams?.[0]?.lastIngestionTime,
})),
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed({
exists: false,
lastIngestion: undefined as number | undefined,
}),
),
);
const now = yield* Effect.sync(() => Date.now());
const quiescent =
observed.exists &&
(observed.lastIngestion === undefined || now - observed.lastIngestion > 120_000);
if (quiescent) {
yield* reapLogGroup;
} else {
yield* reapLogGroup.pipe(
Effect.repeat({
schedule: Schedule.spaced("20 seconds"),
times: 2,
}),
);
}
}
}),
});
}),
);