sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ AppFlow/ Flow.ts

import * as appflow from "@distilled.cloud/aws/appflow";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
import { readAppFlowTags, syncAppFlowTags } from "./internal.ts";

export interface FlowProps {
  /**
   * The name of the flow. Must be unique in the account/region, may contain
   * only letters, digits, hyphens, underscores, `!`, `@`, `#`, `.`, `

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	.repos/alchemy-effect/packages/aws/src/AppFlow/Flow.ts at main · sailfork/t3code-github · Sailfork
	
		
. * If omitted, a deterministic physical name is generated. Changing the * name replaces the flow. */ flowName?: string; /** * A description of the flow. */ description?: string; /** * The ARN of a KMS key AppFlow uses to encrypt data. If omitted, the * AWS-managed AppFlow key is used. */ kmsArn?: string; /** * How the flow is triggered — on demand, on a schedule, or by an event. * @default { triggerType: "OnDemand" } */ triggerConfig?: appflow.TriggerConfig; /** * Configuration of the source connector (e.g. S3, Salesforce) the flow * pulls data from. */ sourceFlowConfig: appflow.SourceFlowConfig; /** * Configuration of the destination connector(s) the flow pushes data to. */ destinationFlowConfigList: appflow.DestinationFlowConfig[]; /** * The field mapping tasks applied to records as they move from source to * destination. */ tasks: appflow.Task[]; /** * Glue Data Catalog configuration for cataloging flow output. */ metadataCatalogConfig?: appflow.MetadataCatalogConfig; /** * User-defined tags for the flow. */ tags?: Record<string, string>; } export interface Flow extends Resource< "AWS.AppFlow.Flow", FlowProps, { flowName: string; flowArn: string; flowStatus: string | undefined; }, never, Providers > {} /** * An Amazon AppFlow flow. A flow transfers data between a source connector * and one or more destination connectors, applying field-mapping tasks. * The credential-free S3-to-S3 path is directly testable; other connectors * require a {@link ConnectorProfile} with vendor credentials. * * For an S3 source, the bucket policy must authorize the * `appflow.amazonaws.com` service principal (`s3:GetObject` + `s3:ListBucket` * on the source; `s3:PutObject` and the multipart/ACL actions on the * destination), and the source prefix must contain at least one object when * the flow is created — AppFlow validates connectivity by listing the source * at `CreateFlow` time and rejects an empty prefix with * `ConnectorServerException`. * ### Creating a Flow * **Example:** S3 to S3 On-Demand Flow * ```typescript * const flow = yield* AppFlow.Flow("Copy", { * triggerConfig: { triggerType: "OnDemand" }, * sourceFlowConfig: { * connectorType: "S3", * sourceConnectorProperties: { * S3: { bucketName: srcBucket, bucketPrefix: "input" }, * }, * }, * destinationFlowConfigList: [ * { * connectorType: "S3", * destinationConnectorProperties: { * S3: { bucketName: dstBucket, bucketPrefix: "output" }, * }, * }, * ], * tasks: [ * { * taskType: "Map_all", * sourceFields: [], * connectorOperator: { S3: "NO_OP" }, * taskProperties: {}, * }, * ], * }); * ``` * * @resource */ export const Flow = Resource<Flow>("AWS.AppFlow.Flow"); const ON_DEMAND: appflow.TriggerConfig = { triggerType: "OnDemand" }; export const FlowProvider = () => Provider.effect( Flow, Effect.gen(function* () { const toName = (id: string, props: FlowProps) => props.flowName ? Effect.succeed(props.flowName) : createPhysicalName({ id, maxLength: 100 }); return { stables: ["flowName", "flowArn"], diff: Effect.fn(function* ({ id, olds, news }) { if (!isResolved(news)) return undefined; if ((yield* toName(id, olds)) !== (yield* toName(id, news))) { return { action: "replace" } as const; } }), read: Effect.fn(function* ({ id, olds, output }) { const name = output?.flowName ?? (yield* toName(id, olds ?? {})); const flow = yield* appflow .describeFlow({ flowName: name }) .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined)); if (flow === undefined || flow.flowArn === undefined) return undefined; const attrs = { flowName: name, flowArn: flow.flowArn, flowStatus: flow.flowStatus, }; const tags = yield* readAppFlowTags(flow.flowArn); return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs); }), reconcile: Effect.fn(function* ({ id, news, output, session }) { const name = output?.flowName ?? (yield* toName(id, news)); const internalTags = yield* createInternalTags(id); const desiredTags = { ...internalTags, ...news.tags }; const trigger = news.triggerConfig ?? ON_DEMAND; // 1. Observe. let live = yield* appflow .describeFlow({ flowName: name }) .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined)); // 2. Ensure — create if missing. Tolerate an AlreadyExists race by // falling back to describe. if (live === undefined) { yield* appflow .createFlow({ flowName: name, description: news.description, kmsArn: news.kmsArn, triggerConfig: trigger, sourceFlowConfig: news.sourceFlowConfig, destinationFlowConfigList: news.destinationFlowConfigList, tasks: news.tasks, metadataCatalogConfig: news.metadataCatalogConfig, tags: desiredTags, }) .pipe(Effect.catchTag("ConflictException", () => Effect.void)); live = yield* appflow.describeFlow({ flowName: name }); } else { // 3. Sync — updateFlow converges the mutable configuration // (trigger, source/destination, tasks). It does not accept tags. yield* appflow.updateFlow({ flowName: name, description: news.description, triggerConfig: trigger, sourceFlowConfig: news.sourceFlowConfig, destinationFlowConfigList: news.destinationFlowConfigList, tasks: news.tasks, metadataCatalogConfig: news.metadataCatalogConfig, }); live = yield* appflow.describeFlow({ flowName: name }); } // 3b. Sync tags against observed cloud tags. if (live.flowArn) { yield* syncAppFlowTags(live.flowArn, desiredTags); } yield* session.note(name); return { flowName: name, flowArn: live.flowArn!, flowStatus: live.flowStatus, }; }), delete: Effect.fn(function* ({ output }) { yield* appflow .deleteFlow({ flowName: output.flowName, forceDelete: true }) .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)); }), list: () => appflow.listFlows.pages({}).pipe( Stream.runCollect, Effect.map((chunk) => Array.from(chunk).flatMap((page) => (page.flows ?? []) .filter( ( f, ): f is appflow.FlowDefinition & { flowName: string; flowArn: string; } => f.flowName !== undefined && f.flowArn !== undefined, ) .map((f) => ({ flowName: f.flowName, flowArn: f.flowArn, flowStatus: f.flowStatus, })), ), ), ), }; }), );
.repos/alchemy-effect/packages/aws/src/AppFlow/Flow.ts at main · sailfork/t3code-github · Sailfork
.\n * If omitted, a deterministic physical name is generated. Changing the\n * name replaces the flow.\n */\n flowName?: string;\n /**\n * A description of the flow.\n */\n description?: string;\n /**\n * The ARN of a KMS key AppFlow uses to encrypt data. If omitted, the\n * AWS-managed AppFlow key is used.\n */\n kmsArn?: string;\n /**\n * How the flow is triggered — on demand, on a schedule, or by an event.\n * @default { triggerType: \"OnDemand\" }\n */\n triggerConfig?: appflow.TriggerConfig;\n /**\n * Configuration of the source connector (e.g. S3, Salesforce) the flow\n * pulls data from.\n */\n sourceFlowConfig: appflow.SourceFlowConfig;\n /**\n * Configuration of the destination connector(s) the flow pushes data to.\n */\n destinationFlowConfigList: appflow.DestinationFlowConfig[];\n /**\n * The field mapping tasks applied to records as they move from source to\n * destination.\n */\n tasks: appflow.Task[];\n /**\n * Glue Data Catalog configuration for cataloging flow output.\n */\n metadataCatalogConfig?: appflow.MetadataCatalogConfig;\n /**\n * User-defined tags for the flow.\n */\n tags?: Record\u003cstring, string>;\n}\n\nexport interface Flow extends Resource\u003c\n \"AWS.AppFlow.Flow\",\n FlowProps,\n {\n flowName: string;\n flowArn: string;\n flowStatus: string | undefined;\n },\n never,\n Providers\n> {}\n\n/**\n * An Amazon AppFlow flow. A flow transfers data between a source connector\n * and one or more destination connectors, applying field-mapping tasks.\n * The credential-free S3-to-S3 path is directly testable; other connectors\n * require a {@link ConnectorProfile} with vendor credentials.\n *\n * For an S3 source, the bucket policy must authorize the\n * `appflow.amazonaws.com` service principal (`s3:GetObject` + `s3:ListBucket`\n * on the source; `s3:PutObject` and the multipart/ACL actions on the\n * destination), and the source prefix must contain at least one object when\n * the flow is created — AppFlow validates connectivity by listing the source\n * at `CreateFlow` time and rejects an empty prefix with\n * `ConnectorServerException`.\n * ### Creating a Flow\n * **Example:** S3 to S3 On-Demand Flow\n * ```typescript\n * const flow = yield* AppFlow.Flow(\"Copy\", {\n * triggerConfig: { triggerType: \"OnDemand\" },\n * sourceFlowConfig: {\n * connectorType: \"S3\",\n * sourceConnectorProperties: {\n * S3: { bucketName: srcBucket, bucketPrefix: \"input\" },\n * },\n * },\n * destinationFlowConfigList: [\n * {\n * connectorType: \"S3\",\n * destinationConnectorProperties: {\n * S3: { bucketName: dstBucket, bucketPrefix: \"output\" },\n * },\n * },\n * ],\n * tasks: [\n * {\n * taskType: \"Map_all\",\n * sourceFields: [],\n * connectorOperator: { S3: \"NO_OP\" },\n * taskProperties: {},\n * },\n * ],\n * });\n * ```\n *\n * @resource\n */\nexport const Flow = Resource\u003cFlow>(\"AWS.AppFlow.Flow\");\n\nconst ON_DEMAND: appflow.TriggerConfig = { triggerType: \"OnDemand\" };\n\nexport const FlowProvider = () =>\n Provider.effect(\n Flow,\n Effect.gen(function* () {\n const toName = (id: string, props: FlowProps) =>\n props.flowName\n ? Effect.succeed(props.flowName)\n : createPhysicalName({ id, maxLength: 100 });\n\n return {\n stables: [\"flowName\", \"flowArn\"],\n\n diff: Effect.fn(function* ({ id, olds, news }) {\n if (!isResolved(news)) return undefined;\n if ((yield* toName(id, olds)) !== (yield* toName(id, news))) {\n return { action: \"replace\" } as const;\n }\n }),\n\n read: Effect.fn(function* ({ id, olds, output }) {\n const name = output?.flowName ?? (yield* toName(id, olds ?? {}));\n const flow = yield* appflow\n .describeFlow({ flowName: name })\n .pipe(Effect.catchTag(\"ResourceNotFoundException\", () => Effect.undefined));\n if (flow === undefined || flow.flowArn === undefined) return undefined;\n const attrs = {\n flowName: name,\n flowArn: flow.flowArn,\n flowStatus: flow.flowStatus,\n };\n const tags = yield* readAppFlowTags(flow.flowArn);\n return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);\n }),\n\n reconcile: Effect.fn(function* ({ id, news, output, session }) {\n const name = output?.flowName ?? (yield* toName(id, news));\n const internalTags = yield* createInternalTags(id);\n const desiredTags = { ...internalTags, ...news.tags };\n const trigger = news.triggerConfig ?? ON_DEMAND;\n\n // 1. Observe.\n let live = yield* appflow\n .describeFlow({ flowName: name })\n .pipe(Effect.catchTag(\"ResourceNotFoundException\", () => Effect.undefined));\n\n // 2. Ensure — create if missing. Tolerate an AlreadyExists race by\n // falling back to describe.\n if (live === undefined) {\n yield* appflow\n .createFlow({\n flowName: name,\n description: news.description,\n kmsArn: news.kmsArn,\n triggerConfig: trigger,\n sourceFlowConfig: news.sourceFlowConfig,\n destinationFlowConfigList: news.destinationFlowConfigList,\n tasks: news.tasks,\n metadataCatalogConfig: news.metadataCatalogConfig,\n tags: desiredTags,\n })\n .pipe(Effect.catchTag(\"ConflictException\", () => Effect.void));\n live = yield* appflow.describeFlow({ flowName: name });\n } else {\n // 3. Sync — updateFlow converges the mutable configuration\n // (trigger, source/destination, tasks). It does not accept tags.\n yield* appflow.updateFlow({\n flowName: name,\n description: news.description,\n triggerConfig: trigger,\n sourceFlowConfig: news.sourceFlowConfig,\n destinationFlowConfigList: news.destinationFlowConfigList,\n tasks: news.tasks,\n metadataCatalogConfig: news.metadataCatalogConfig,\n });\n live = yield* appflow.describeFlow({ flowName: name });\n }\n\n // 3b. Sync tags against observed cloud tags.\n if (live.flowArn) {\n yield* syncAppFlowTags(live.flowArn, desiredTags);\n }\n\n yield* session.note(name);\n return {\n flowName: name,\n flowArn: live.flowArn!,\n flowStatus: live.flowStatus,\n };\n }),\n\n delete: Effect.fn(function* ({ output }) {\n yield* appflow\n .deleteFlow({ flowName: output.flowName, forceDelete: true })\n .pipe(Effect.catchTag(\"ResourceNotFoundException\", () => Effect.void));\n }),\n\n list: () =>\n appflow.listFlows.pages({}).pipe(\n Stream.runCollect,\n Effect.map((chunk) =>\n Array.from(chunk).flatMap((page) =>\n (page.flows ?? [])\n .filter(\n (\n f,\n ): f is appflow.FlowDefinition & {\n flowName: string;\n flowArn: string;\n } => f.flowName !== undefined && f.flowArn !== undefined,\n )\n .map((f) => ({\n flowName: f.flowName,\n flowArn: f.flowArn,\n flowStatus: f.flowStatus,\n })),\n ),\n ),\n ),\n };\n }),\n );\n","preview":null,"history":null,"version":"918171cc113d76075cfab6c6da2ce61ea362ebb0"},"/_session?shell=1":null}