sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Translate/ ParallelData.ts

import * as translate from "@distilled.cloud/aws/translate";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
import { readTranslateTags, syncTranslateTags } from "./internal.ts";

/**
 * The parallel data import failed server-side (`Status: FAILED`) — usually a
 * malformed input file or an S3 URI the caller cannot read.
 */
export class ParallelDataFailed extends Data.TaggedError("ParallelDataFailed")<{
  readonly resourceName: string;
  readonly message: string | undefined;
}> {}

/**
 * The parallel data did not reach a terminal status within the bounded
 * polling window.
 */
export class ParallelDataNotConverged extends Data.TaggedError("ParallelDataNotConverged")<{
  readonly resourceName: string;
  readonly status: string | undefined;
}> {}

export interface ParallelDataProps {
  /**
   * Name of the parallel data — up to 256 characters of letters, digits, and
   * hyphens (`^([A-Za-z0-9-]_?)+

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	.repos/alchemy-effect/packages/aws/src/Translate/ParallelData.ts at main · sailfork/t3code-github · Sailfork
	
		
). If omitted, a unique name is generated * from the app, stage, and logical ID. Changing the name replaces the * parallel data. */ parallelDataName?: string; /** * Description of the parallel data. */ description?: string; /** * S3 URI of the parallel data input file (e.g. * `s3://my-bucket/examples.csv`). The caller must be able to read the * object. Updatable in place — a new import runs on change. */ s3Uri: string; /** * Format of the input file: `TSV`, `CSV`, or `TMX`. */ format?: "TSV" | "CSV" | "TMX"; /** * Id of the customer-managed KMS key used to encrypt the parallel data. * If omitted, Translate uses an AWS-owned key. */ encryptionKeyId?: string; /** * User-defined tags for the parallel data. */ tags?: Record<string, string>; } export interface ParallelData extends Resource< "AWS.Translate.ParallelData", ParallelDataProps, { /** * Name of the parallel data — pass it as `ParallelDataNames` to * `StartTextTranslationJob` for Active Custom Translation. */ parallelDataName: string; /** * ARN of the parallel data, e.g. * `arn:aws:translate:us-east-1:123456789012:parallel-data/style-examples`. */ parallelDataArn: string; /** Status of the parallel data (`ACTIVE` once import succeeds). */ status: string | undefined; /** Source language code detected from the input file, e.g. `en`. */ sourceLanguageCode: string | undefined; /** Target language codes detected from the input file. */ targetLanguageCodes: string[] | undefined; /** Number of records imported. */ importedRecordCount: number | undefined; /** Number of records that failed to import. */ failedRecordCount: number | undefined; }, never, Providers > {} /** * Amazon Translate parallel data — segment-aligned translation examples * imported from S3 that steer the style, tone, and word choice of batch * translation jobs (Active Custom Translation). The import is asynchronous: * the resource waits for the parallel data to become `ACTIVE`. * * ### Managing Parallel Data * **Example:** Import parallel data from S3 * ```typescript * const examples = yield* AWS.Translate.ParallelData("StyleExamples", { * s3Uri: "s3://my-bucket/style-examples.csv", * format: "CSV", * }); * ``` * * **Example:** Use parallel data in a batch translation job * ```typescript * const startJob = yield* AWS.Translate.StartTextTranslationJob(dataAccessRole); * yield* startJob({ * InputDataConfig: { S3Uri: "s3://my-bucket/input/", ContentType: "text/plain" }, * OutputDataConfig: { S3Uri: "s3://my-bucket/output/" }, * SourceLanguageCode: "en", * TargetLanguageCodes: ["es"], * ParallelDataNames: [examples.parallelDataName], * }); * ``` * * @resource */ export const ParallelData = Resource<ParallelData>("AWS.Translate.ParallelData"); const toAttributes = (props: translate.ParallelDataProperties) => ({ parallelDataName: props.Name!, parallelDataArn: props.Arn!, status: props.Status, sourceLanguageCode: props.SourceLanguageCode, targetLanguageCodes: props.TargetLanguageCodes ? [...props.TargetLanguageCodes] : undefined, importedRecordCount: props.ImportedRecordCount, failedRecordCount: props.FailedRecordCount, }); export const ParallelDataProvider = () => Provider.effect( ParallelData, Effect.gen(function* () { const createName = Effect.fn(function* (id: string, props: ParallelDataProps) { return props.parallelDataName ?? (yield* createPhysicalName({ id, maxLength: 256 })); }); const getOne = Effect.fn(function* (name: string) { return yield* translate .getParallelData({ Name: name }) .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined)); }); // The import runs asynchronously (CREATING/UPDATING → ACTIVE|FAILED); // poll bounded (~10 min ceiling — small files typically converge in a // few minutes). const waitForStable = Effect.fn(function* (name: string) { const final = yield* getOne(name).pipe( Effect.repeat({ schedule: Schedule.spaced("10 seconds"), until: (r): boolean => { const status = r?.ParallelDataProperties?.Status; return r === undefined || status === "ACTIVE" || status === "FAILED"; }, times: 60, }), ); const props = final?.ParallelDataProperties; if (props?.Status === "FAILED") { return yield* new ParallelDataFailed({ resourceName: name, message: props.Message }); } if (props?.Status !== "ACTIVE") { return yield* new ParallelDataNotConverged({ resourceName: name, status: props?.Status }); } return props; }); return { stables: ["parallelDataName", "parallelDataArn"], diff: Effect.fn(function* ({ id, olds, news }) { if (!isResolved(news)) return undefined; if ((yield* createName(id, olds)) !== (yield* createName(id, news))) { return { action: "replace" } as const; } }), read: Effect.fn(function* ({ id, olds, output }) { const name = output?.parallelDataName ?? (yield* createName(id, olds ?? { s3Uri: "" })); const found = yield* getOne(name); if (found?.ParallelDataProperties === undefined) return undefined; const attrs = toAttributes(found.ParallelDataProperties); const tags = yield* readTranslateTags(attrs.parallelDataArn); return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs); }), reconcile: Effect.fn(function* ({ id, news, output, session }) { const name = output?.parallelDataName ?? (yield* createName(id, news)); const internalTags = yield* createInternalTags(id); const desiredTags = { ...internalTags, ...news.tags }; // Observe — cloud state is authoritative. const observed = yield* getOne(name); const observedProps = observed?.ParallelDataProperties; if (observedProps === undefined) { // Ensure — greenfield import. ConflictException is a race with a // concurrent create of the same name; fall through to observe. // The client token only needs to be unique per submission; // idempotency across crashed runs comes from observing first. const clientToken = yield* Effect.sync(() => crypto.randomUUID()); yield* translate .createParallelData({ Name: name, ClientToken: clientToken, Description: news.description, ParallelDataConfig: { S3Uri: news.s3Uri, Format: news.format }, EncryptionKey: news.encryptionKeyId ? { Type: "KMS", Id: news.encryptionKeyId } : undefined, Tags: Object.entries(desiredTags).map(([Key, Value]) => ({ Key, Value, })), }) .pipe(Effect.catchTag("ConflictException", () => Effect.void)); } else { // Sync — diff the observed import config against the desired one // and re-import only on drift. const config = observedProps.ParallelDataConfig; const drift = config?.S3Uri !== news.s3Uri || (news.format !== undefined && config?.Format !== news.format) || (observedProps.Description ?? undefined) !== (news.description ?? undefined); if (drift) { // Computed outside the retry so replays of the same submission // reuse one idempotency token. const clientToken = yield* Effect.sync(() => crypto.randomUUID()); yield* translate .updateParallelData({ Name: name, ClientToken: clientToken, Description: news.description, ParallelDataConfig: { S3Uri: news.s3Uri, Format: news.format, }, }) .pipe( // A concurrent import may still be settling — retry bounded. Effect.retry({ while: (e): boolean => e._tag === "ConcurrentModificationException" || e._tag === "ConflictException", schedule: Schedule.spaced("10 seconds"), times: 8, }), ); } } yield* session.note(name); const props = yield* waitForStable(name); yield* syncTranslateTags(props.Arn!, desiredTags); return toAttributes(props); }), delete: Effect.fn(function* ({ output }) { // Idempotent — the parallel data may already be gone. A delete // during CREATING/UPDATING conflicts; retry bounded. yield* translate.deleteParallelData({ Name: output.parallelDataName }).pipe( Effect.retry({ while: (e): boolean => e._tag === "ConcurrentModificationException", schedule: Schedule.spaced("10 seconds"), times: 8, }), Effect.catchTag("ResourceNotFoundException", () => Effect.void), ); // Deletion is asynchronous (DELETING) — wait until gone so a // replacement can immediately reuse the name. yield* translate.getParallelData({ Name: output.parallelDataName }).pipe( Effect.catchTag("ResourceNotFoundException", () => Effect.undefined), Effect.repeat({ schedule: Schedule.spaced("5 seconds"), until: (r): boolean => r === undefined, times: 36, }), ); }), list: () => translate.listParallelData.pages({}).pipe( Stream.runCollect, Effect.map((chunk) => Array.from(chunk) .flatMap((page) => page.ParallelDataPropertiesList ?? []) .filter((props) => props.Name && props.Arn) .map(toAttributes), ), ), }; }), );
.repos/alchemy-effect/packages/aws/src/Translate/ParallelData.ts at main · sailfork/t3code-github · Sailfork
). If omitted, a unique name is generated\n * from the app, stage, and logical ID. Changing the name replaces the\n * parallel data.\n */\n parallelDataName?: string;\n /**\n * Description of the parallel data.\n */\n description?: string;\n /**\n * S3 URI of the parallel data input file (e.g.\n * `s3://my-bucket/examples.csv`). The caller must be able to read the\n * object. Updatable in place — a new import runs on change.\n */\n s3Uri: string;\n /**\n * Format of the input file: `TSV`, `CSV`, or `TMX`.\n */\n format?: \"TSV\" | \"CSV\" | \"TMX\";\n /**\n * Id of the customer-managed KMS key used to encrypt the parallel data.\n * If omitted, Translate uses an AWS-owned key.\n */\n encryptionKeyId?: string;\n /**\n * User-defined tags for the parallel data.\n */\n tags?: Record\u003cstring, string>;\n}\n\nexport interface ParallelData extends Resource\u003c\n \"AWS.Translate.ParallelData\",\n ParallelDataProps,\n {\n /**\n * Name of the parallel data — pass it as `ParallelDataNames` to\n * `StartTextTranslationJob` for Active Custom Translation.\n */\n parallelDataName: string;\n /**\n * ARN of the parallel data, e.g.\n * `arn:aws:translate:us-east-1:123456789012:parallel-data/style-examples`.\n */\n parallelDataArn: string;\n /** Status of the parallel data (`ACTIVE` once import succeeds). */\n status: string | undefined;\n /** Source language code detected from the input file, e.g. `en`. */\n sourceLanguageCode: string | undefined;\n /** Target language codes detected from the input file. */\n targetLanguageCodes: string[] | undefined;\n /** Number of records imported. */\n importedRecordCount: number | undefined;\n /** Number of records that failed to import. */\n failedRecordCount: number | undefined;\n },\n never,\n Providers\n> {}\n\n/**\n * Amazon Translate parallel data — segment-aligned translation examples\n * imported from S3 that steer the style, tone, and word choice of batch\n * translation jobs (Active Custom Translation). The import is asynchronous:\n * the resource waits for the parallel data to become `ACTIVE`.\n *\n * ### Managing Parallel Data\n * **Example:** Import parallel data from S3\n * ```typescript\n * const examples = yield* AWS.Translate.ParallelData(\"StyleExamples\", {\n * s3Uri: \"s3://my-bucket/style-examples.csv\",\n * format: \"CSV\",\n * });\n * ```\n *\n * **Example:** Use parallel data in a batch translation job\n * ```typescript\n * const startJob = yield* AWS.Translate.StartTextTranslationJob(dataAccessRole);\n * yield* startJob({\n * InputDataConfig: { S3Uri: \"s3://my-bucket/input/\", ContentType: \"text/plain\" },\n * OutputDataConfig: { S3Uri: \"s3://my-bucket/output/\" },\n * SourceLanguageCode: \"en\",\n * TargetLanguageCodes: [\"es\"],\n * ParallelDataNames: [examples.parallelDataName],\n * });\n * ```\n *\n * @resource\n */\nexport const ParallelData = Resource\u003cParallelData>(\"AWS.Translate.ParallelData\");\n\nconst toAttributes = (props: translate.ParallelDataProperties) => ({\n parallelDataName: props.Name!,\n parallelDataArn: props.Arn!,\n status: props.Status,\n sourceLanguageCode: props.SourceLanguageCode,\n targetLanguageCodes: props.TargetLanguageCodes ? [...props.TargetLanguageCodes] : undefined,\n importedRecordCount: props.ImportedRecordCount,\n failedRecordCount: props.FailedRecordCount,\n});\n\nexport const ParallelDataProvider = () =>\n Provider.effect(\n ParallelData,\n Effect.gen(function* () {\n const createName = Effect.fn(function* (id: string, props: ParallelDataProps) {\n return props.parallelDataName ?? (yield* createPhysicalName({ id, maxLength: 256 }));\n });\n\n const getOne = Effect.fn(function* (name: string) {\n return yield* translate\n .getParallelData({ Name: name })\n .pipe(Effect.catchTag(\"ResourceNotFoundException\", () => Effect.undefined));\n });\n\n // The import runs asynchronously (CREATING/UPDATING → ACTIVE|FAILED);\n // poll bounded (~10 min ceiling — small files typically converge in a\n // few minutes).\n const waitForStable = Effect.fn(function* (name: string) {\n const final = yield* getOne(name).pipe(\n Effect.repeat({\n schedule: Schedule.spaced(\"10 seconds\"),\n until: (r): boolean => {\n const status = r?.ParallelDataProperties?.Status;\n return r === undefined || status === \"ACTIVE\" || status === \"FAILED\";\n },\n times: 60,\n }),\n );\n const props = final?.ParallelDataProperties;\n if (props?.Status === \"FAILED\") {\n return yield* new ParallelDataFailed({ resourceName: name, message: props.Message });\n }\n if (props?.Status !== \"ACTIVE\") {\n return yield* new ParallelDataNotConverged({ resourceName: name, status: props?.Status });\n }\n return props;\n });\n\n return {\n stables: [\"parallelDataName\", \"parallelDataArn\"],\n\n diff: Effect.fn(function* ({ id, olds, news }) {\n if (!isResolved(news)) return undefined;\n if ((yield* createName(id, olds)) !== (yield* createName(id, news))) {\n return { action: \"replace\" } as const;\n }\n }),\n\n read: Effect.fn(function* ({ id, olds, output }) {\n const name = output?.parallelDataName ?? (yield* createName(id, olds ?? { s3Uri: \"\" }));\n const found = yield* getOne(name);\n if (found?.ParallelDataProperties === undefined) return undefined;\n const attrs = toAttributes(found.ParallelDataProperties);\n const tags = yield* readTranslateTags(attrs.parallelDataArn);\n return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);\n }),\n\n reconcile: Effect.fn(function* ({ id, news, output, session }) {\n const name = output?.parallelDataName ?? (yield* createName(id, news));\n const internalTags = yield* createInternalTags(id);\n const desiredTags = { ...internalTags, ...news.tags };\n\n // Observe — cloud state is authoritative.\n const observed = yield* getOne(name);\n const observedProps = observed?.ParallelDataProperties;\n\n if (observedProps === undefined) {\n // Ensure — greenfield import. ConflictException is a race with a\n // concurrent create of the same name; fall through to observe.\n // The client token only needs to be unique per submission;\n // idempotency across crashed runs comes from observing first.\n const clientToken = yield* Effect.sync(() => crypto.randomUUID());\n yield* translate\n .createParallelData({\n Name: name,\n ClientToken: clientToken,\n Description: news.description,\n ParallelDataConfig: { S3Uri: news.s3Uri, Format: news.format },\n EncryptionKey: news.encryptionKeyId\n ? { Type: \"KMS\", Id: news.encryptionKeyId }\n : undefined,\n Tags: Object.entries(desiredTags).map(([Key, Value]) => ({\n Key,\n Value,\n })),\n })\n .pipe(Effect.catchTag(\"ConflictException\", () => Effect.void));\n } else {\n // Sync — diff the observed import config against the desired one\n // and re-import only on drift.\n const config = observedProps.ParallelDataConfig;\n const drift =\n config?.S3Uri !== news.s3Uri ||\n (news.format !== undefined && config?.Format !== news.format) ||\n (observedProps.Description ?? undefined) !== (news.description ?? undefined);\n if (drift) {\n // Computed outside the retry so replays of the same submission\n // reuse one idempotency token.\n const clientToken = yield* Effect.sync(() => crypto.randomUUID());\n yield* translate\n .updateParallelData({\n Name: name,\n ClientToken: clientToken,\n Description: news.description,\n ParallelDataConfig: {\n S3Uri: news.s3Uri,\n Format: news.format,\n },\n })\n .pipe(\n // A concurrent import may still be settling — retry bounded.\n Effect.retry({\n while: (e): boolean =>\n e._tag === \"ConcurrentModificationException\" ||\n e._tag === \"ConflictException\",\n schedule: Schedule.spaced(\"10 seconds\"),\n times: 8,\n }),\n );\n }\n }\n\n yield* session.note(name);\n const props = yield* waitForStable(name);\n yield* syncTranslateTags(props.Arn!, desiredTags);\n return toAttributes(props);\n }),\n\n delete: Effect.fn(function* ({ output }) {\n // Idempotent — the parallel data may already be gone. A delete\n // during CREATING/UPDATING conflicts; retry bounded.\n yield* translate.deleteParallelData({ Name: output.parallelDataName }).pipe(\n Effect.retry({\n while: (e): boolean => e._tag === \"ConcurrentModificationException\",\n schedule: Schedule.spaced(\"10 seconds\"),\n times: 8,\n }),\n Effect.catchTag(\"ResourceNotFoundException\", () => Effect.void),\n );\n // Deletion is asynchronous (DELETING) — wait until gone so a\n // replacement can immediately reuse the name.\n yield* translate.getParallelData({ Name: output.parallelDataName }).pipe(\n Effect.catchTag(\"ResourceNotFoundException\", () => Effect.undefined),\n Effect.repeat({\n schedule: Schedule.spaced(\"5 seconds\"),\n until: (r): boolean => r === undefined,\n times: 36,\n }),\n );\n }),\n\n list: () =>\n translate.listParallelData.pages({}).pipe(\n Stream.runCollect,\n Effect.map((chunk) =>\n Array.from(chunk)\n .flatMap((page) => page.ParallelDataPropertiesList ?? [])\n .filter((props) => props.Name && props.Arn)\n .map(toAttributes),\n ),\n ),\n };\n }),\n );\n","preview":null,"history":null,"version":"0b08ced148ad3954a92880653fcb1948cbf412ca"},"/_session?shell=1":null}