import * as translate from "@distilled.cloud/aws/translate";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
import { readTranslateTags, syncTranslateTags } from "./internal.ts";
/**
* The parallel data import failed server-side (`Status: FAILED`) — usually a
* malformed input file or an S3 URI the caller cannot read.
*/
export class ParallelDataFailed extends Data.TaggedError("ParallelDataFailed")<{
readonly resourceName: string;
readonly message: string | undefined;
}> {}
/**
* The parallel data did not reach a terminal status within the bounded
* polling window.
*/
export class ParallelDataNotConverged extends Data.TaggedError("ParallelDataNotConverged")<{
readonly resourceName: string;
readonly status: string | undefined;
}> {}
export interface ParallelDataProps {
/**
* Name of the parallel data — up to 256 characters of letters, digits, and
* hyphens (`^([A-Za-z0-9-]_?)+
.repos/alchemy-effect/packages/aws/src/Translate/ParallelData.ts at main · sailfork/t3code-github · Sailfork
). If omitted, a unique name is generated
* from the app, stage, and logical ID. Changing the name replaces the
* parallel data.
*/
parallelDataName?: string;
/**
* Description of the parallel data.
*/
description?: string;
/**
* S3 URI of the parallel data input file (e.g.
* `s3://my-bucket/examples.csv`). The caller must be able to read the
* object. Updatable in place — a new import runs on change.
*/
s3Uri: string;
/**
* Format of the input file: `TSV`, `CSV`, or `TMX`.
*/
format?: "TSV" | "CSV" | "TMX";
/**
* Id of the customer-managed KMS key used to encrypt the parallel data.
* If omitted, Translate uses an AWS-owned key.
*/
encryptionKeyId?: string;
/**
* User-defined tags for the parallel data.
*/
tags?: Record<string, string>;
}
export interface ParallelData extends Resource<
"AWS.Translate.ParallelData",
ParallelDataProps,
{
/**
* Name of the parallel data — pass it as `ParallelDataNames` to
* `StartTextTranslationJob` for Active Custom Translation.
*/
parallelDataName: string;
/**
* ARN of the parallel data, e.g.
* `arn:aws:translate:us-east-1:123456789012:parallel-data/style-examples`.
*/
parallelDataArn: string;
/** Status of the parallel data (`ACTIVE` once import succeeds). */
status: string | undefined;
/** Source language code detected from the input file, e.g. `en`. */
sourceLanguageCode: string | undefined;
/** Target language codes detected from the input file. */
targetLanguageCodes: string[] | undefined;
/** Number of records imported. */
importedRecordCount: number | undefined;
/** Number of records that failed to import. */
failedRecordCount: number | undefined;
},
never,
Providers
> {}
/**
* Amazon Translate parallel data — segment-aligned translation examples
* imported from S3 that steer the style, tone, and word choice of batch
* translation jobs (Active Custom Translation). The import is asynchronous:
* the resource waits for the parallel data to become `ACTIVE`.
*
* ### Managing Parallel Data
* **Example:** Import parallel data from S3
* ```typescript
* const examples = yield* AWS.Translate.ParallelData("StyleExamples", {
* s3Uri: "s3://my-bucket/style-examples.csv",
* format: "CSV",
* });
* ```
*
* **Example:** Use parallel data in a batch translation job
* ```typescript
* const startJob = yield* AWS.Translate.StartTextTranslationJob(dataAccessRole);
* yield* startJob({
* InputDataConfig: { S3Uri: "s3://my-bucket/input/", ContentType: "text/plain" },
* OutputDataConfig: { S3Uri: "s3://my-bucket/output/" },
* SourceLanguageCode: "en",
* TargetLanguageCodes: ["es"],
* ParallelDataNames: [examples.parallelDataName],
* });
* ```
*
* @resource
*/
export const ParallelData = Resource<ParallelData>("AWS.Translate.ParallelData");
const toAttributes = (props: translate.ParallelDataProperties) => ({
parallelDataName: props.Name!,
parallelDataArn: props.Arn!,
status: props.Status,
sourceLanguageCode: props.SourceLanguageCode,
targetLanguageCodes: props.TargetLanguageCodes ? [...props.TargetLanguageCodes] : undefined,
importedRecordCount: props.ImportedRecordCount,
failedRecordCount: props.FailedRecordCount,
});
export const ParallelDataProvider = () =>
Provider.effect(
ParallelData,
Effect.gen(function* () {
const createName = Effect.fn(function* (id: string, props: ParallelDataProps) {
return props.parallelDataName ?? (yield* createPhysicalName({ id, maxLength: 256 }));
});
const getOne = Effect.fn(function* (name: string) {
return yield* translate
.getParallelData({ Name: name })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
});
// The import runs asynchronously (CREATING/UPDATING → ACTIVE|FAILED);
// poll bounded (~10 min ceiling — small files typically converge in a
// few minutes).
const waitForStable = Effect.fn(function* (name: string) {
const final = yield* getOne(name).pipe(
Effect.repeat({
schedule: Schedule.spaced("10 seconds"),
until: (r): boolean => {
const status = r?.ParallelDataProperties?.Status;
return r === undefined || status === "ACTIVE" || status === "FAILED";
},
times: 60,
}),
);
const props = final?.ParallelDataProperties;
if (props?.Status === "FAILED") {
return yield* new ParallelDataFailed({ resourceName: name, message: props.Message });
}
if (props?.Status !== "ACTIVE") {
return yield* new ParallelDataNotConverged({ resourceName: name, status: props?.Status });
}
return props;
});
return {
stables: ["parallelDataName", "parallelDataArn"],
diff: Effect.fn(function* ({ id, olds, news }) {
if (!isResolved(news)) return undefined;
if ((yield* createName(id, olds)) !== (yield* createName(id, news))) {
return { action: "replace" } as const;
}
}),
read: Effect.fn(function* ({ id, olds, output }) {
const name = output?.parallelDataName ?? (yield* createName(id, olds ?? { s3Uri: "" }));
const found = yield* getOne(name);
if (found?.ParallelDataProperties === undefined) return undefined;
const attrs = toAttributes(found.ParallelDataProperties);
const tags = yield* readTranslateTags(attrs.parallelDataArn);
return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
const name = output?.parallelDataName ?? (yield* createName(id, news));
const internalTags = yield* createInternalTags(id);
const desiredTags = { ...internalTags, ...news.tags };
// Observe — cloud state is authoritative.
const observed = yield* getOne(name);
const observedProps = observed?.ParallelDataProperties;
if (observedProps === undefined) {
// Ensure — greenfield import. ConflictException is a race with a
// concurrent create of the same name; fall through to observe.
// The client token only needs to be unique per submission;
// idempotency across crashed runs comes from observing first.
const clientToken = yield* Effect.sync(() => crypto.randomUUID());
yield* translate
.createParallelData({
Name: name,
ClientToken: clientToken,
Description: news.description,
ParallelDataConfig: { S3Uri: news.s3Uri, Format: news.format },
EncryptionKey: news.encryptionKeyId
? { Type: "KMS", Id: news.encryptionKeyId }
: undefined,
Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
Key,
Value,
})),
})
.pipe(Effect.catchTag("ConflictException", () => Effect.void));
} else {
// Sync — diff the observed import config against the desired one
// and re-import only on drift.
const config = observedProps.ParallelDataConfig;
const drift =
config?.S3Uri !== news.s3Uri ||
(news.format !== undefined && config?.Format !== news.format) ||
(observedProps.Description ?? undefined) !== (news.description ?? undefined);
if (drift) {
// Computed outside the retry so replays of the same submission
// reuse one idempotency token.
const clientToken = yield* Effect.sync(() => crypto.randomUUID());
yield* translate
.updateParallelData({
Name: name,
ClientToken: clientToken,
Description: news.description,
ParallelDataConfig: {
S3Uri: news.s3Uri,
Format: news.format,
},
})
.pipe(
// A concurrent import may still be settling — retry bounded.
Effect.retry({
while: (e): boolean =>
e._tag === "ConcurrentModificationException" ||
e._tag === "ConflictException",
schedule: Schedule.spaced("10 seconds"),
times: 8,
}),
);
}
}
yield* session.note(name);
const props = yield* waitForStable(name);
yield* syncTranslateTags(props.Arn!, desiredTags);
return toAttributes(props);
}),
delete: Effect.fn(function* ({ output }) {
// Idempotent — the parallel data may already be gone. A delete
// during CREATING/UPDATING conflicts; retry bounded.
yield* translate.deleteParallelData({ Name: output.parallelDataName }).pipe(
Effect.retry({
while: (e): boolean => e._tag === "ConcurrentModificationException",
schedule: Schedule.spaced("10 seconds"),
times: 8,
}),
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
);
// Deletion is asynchronous (DELETING) — wait until gone so a
// replacement can immediately reuse the name.
yield* translate.getParallelData({ Name: output.parallelDataName }).pipe(
Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
Effect.repeat({
schedule: Schedule.spaced("5 seconds"),
until: (r): boolean => r === undefined,
times: 36,
}),
);
}),
list: () =>
translate.listParallelData.pages({}).pipe(
Stream.runCollect,
Effect.map((chunk) =>
Array.from(chunk)
.flatMap((page) => page.ParallelDataPropertiesList ?? [])
.filter((props) => props.Name && props.Arn)
.map(toAttributes),
),
),
};
}),
);
.repos/alchemy-effect/packages/aws/src/Translate/ParallelData.ts at main · sailfork/t3code-github · Sailfork
). If omitted, a unique name is generated\n * from the app, stage, and logical ID. Changing the name replaces the\n * parallel data.\n */\n parallelDataName?: string;\n /**\n * Description of the parallel data.\n */\n description?: string;\n /**\n * S3 URI of the parallel data input file (e.g.\n * `s3://my-bucket/examples.csv`). The caller must be able to read the\n * object. Updatable in place — a new import runs on change.\n */\n s3Uri: string;\n /**\n * Format of the input file: `TSV`, `CSV`, or `TMX`.\n */\n format?: \"TSV\" | \"CSV\" | \"TMX\";\n /**\n * Id of the customer-managed KMS key used to encrypt the parallel data.\n * If omitted, Translate uses an AWS-owned key.\n */\n encryptionKeyId?: string;\n /**\n * User-defined tags for the parallel data.\n */\n tags?: Record\u003cstring, string>;\n}\n\nexport interface ParallelData extends Resource\u003c\n \"AWS.Translate.ParallelData\",\n ParallelDataProps,\n {\n /**\n * Name of the parallel data — pass it as `ParallelDataNames` to\n * `StartTextTranslationJob` for Active Custom Translation.\n */\n parallelDataName: string;\n /**\n * ARN of the parallel data, e.g.\n * `arn:aws:translate:us-east-1:123456789012:parallel-data/style-examples`.\n */\n parallelDataArn: string;\n /** Status of the parallel data (`ACTIVE` once import succeeds). */\n status: string | undefined;\n /** Source language code detected from the input file, e.g. `en`. */\n sourceLanguageCode: string | undefined;\n /** Target language codes detected from the input file. */\n targetLanguageCodes: string[] | undefined;\n /** Number of records imported. */\n importedRecordCount: number | undefined;\n /** Number of records that failed to import. */\n failedRecordCount: number | undefined;\n },\n never,\n Providers\n> {}\n\n/**\n * Amazon Translate parallel data — segment-aligned translation examples\n * imported from S3 that steer the style, tone, and word choice of batch\n * translation jobs (Active Custom Translation). The import is asynchronous:\n * the resource waits for the parallel data to become `ACTIVE`.\n *\n * ### Managing Parallel Data\n * **Example:** Import parallel data from S3\n * ```typescript\n * const examples = yield* AWS.Translate.ParallelData(\"StyleExamples\", {\n * s3Uri: \"s3://my-bucket/style-examples.csv\",\n * format: \"CSV\",\n * });\n * ```\n *\n * **Example:** Use parallel data in a batch translation job\n * ```typescript\n * const startJob = yield* AWS.Translate.StartTextTranslationJob(dataAccessRole);\n * yield* startJob({\n * InputDataConfig: { S3Uri: \"s3://my-bucket/input/\", ContentType: \"text/plain\" },\n * OutputDataConfig: { S3Uri: \"s3://my-bucket/output/\" },\n * SourceLanguageCode: \"en\",\n * TargetLanguageCodes: [\"es\"],\n * ParallelDataNames: [examples.parallelDataName],\n * });\n * ```\n *\n * @resource\n */\nexport const ParallelData = Resource\u003cParallelData>(\"AWS.Translate.ParallelData\");\n\nconst toAttributes = (props: translate.ParallelDataProperties) => ({\n parallelDataName: props.Name!,\n parallelDataArn: props.Arn!,\n status: props.Status,\n sourceLanguageCode: props.SourceLanguageCode,\n targetLanguageCodes: props.TargetLanguageCodes ? [...props.TargetLanguageCodes] : undefined,\n importedRecordCount: props.ImportedRecordCount,\n failedRecordCount: props.FailedRecordCount,\n});\n\nexport const ParallelDataProvider = () =>\n Provider.effect(\n ParallelData,\n Effect.gen(function* () {\n const createName = Effect.fn(function* (id: string, props: ParallelDataProps) {\n return props.parallelDataName ?? (yield* createPhysicalName({ id, maxLength: 256 }));\n });\n\n const getOne = Effect.fn(function* (name: string) {\n return yield* translate\n .getParallelData({ Name: name })\n .pipe(Effect.catchTag(\"ResourceNotFoundException\", () => Effect.undefined));\n });\n\n // The import runs asynchronously (CREATING/UPDATING → ACTIVE|FAILED);\n // poll bounded (~10 min ceiling — small files typically converge in a\n // few minutes).\n const waitForStable = Effect.fn(function* (name: string) {\n const final = yield* getOne(name).pipe(\n Effect.repeat({\n schedule: Schedule.spaced(\"10 seconds\"),\n until: (r): boolean => {\n const status = r?.ParallelDataProperties?.Status;\n return r === undefined || status === \"ACTIVE\" || status === \"FAILED\";\n },\n times: 60,\n }),\n );\n const props = final?.ParallelDataProperties;\n if (props?.Status === \"FAILED\") {\n return yield* new ParallelDataFailed({ resourceName: name, message: props.Message });\n }\n if (props?.Status !== \"ACTIVE\") {\n return yield* new ParallelDataNotConverged({ resourceName: name, status: props?.Status });\n }\n return props;\n });\n\n return {\n stables: [\"parallelDataName\", \"parallelDataArn\"],\n\n diff: Effect.fn(function* ({ id, olds, news }) {\n if (!isResolved(news)) return undefined;\n if ((yield* createName(id, olds)) !== (yield* createName(id, news))) {\n return { action: \"replace\" } as const;\n }\n }),\n\n read: Effect.fn(function* ({ id, olds, output }) {\n const name = output?.parallelDataName ?? (yield* createName(id, olds ?? { s3Uri: \"\" }));\n const found = yield* getOne(name);\n if (found?.ParallelDataProperties === undefined) return undefined;\n const attrs = toAttributes(found.ParallelDataProperties);\n const tags = yield* readTranslateTags(attrs.parallelDataArn);\n return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs);\n }),\n\n reconcile: Effect.fn(function* ({ id, news, output, session }) {\n const name = output?.parallelDataName ?? (yield* createName(id, news));\n const internalTags = yield* createInternalTags(id);\n const desiredTags = { ...internalTags, ...news.tags };\n\n // Observe — cloud state is authoritative.\n const observed = yield* getOne(name);\n const observedProps = observed?.ParallelDataProperties;\n\n if (observedProps === undefined) {\n // Ensure — greenfield import. ConflictException is a race with a\n // concurrent create of the same name; fall through to observe.\n // The client token only needs to be unique per submission;\n // idempotency across crashed runs comes from observing first.\n const clientToken = yield* Effect.sync(() => crypto.randomUUID());\n yield* translate\n .createParallelData({\n Name: name,\n ClientToken: clientToken,\n Description: news.description,\n ParallelDataConfig: { S3Uri: news.s3Uri, Format: news.format },\n EncryptionKey: news.encryptionKeyId\n ? { Type: \"KMS\", Id: news.encryptionKeyId }\n : undefined,\n Tags: Object.entries(desiredTags).map(([Key, Value]) => ({\n Key,\n Value,\n })),\n })\n .pipe(Effect.catchTag(\"ConflictException\", () => Effect.void));\n } else {\n // Sync — diff the observed import config against the desired one\n // and re-import only on drift.\n const config = observedProps.ParallelDataConfig;\n const drift =\n config?.S3Uri !== news.s3Uri ||\n (news.format !== undefined && config?.Format !== news.format) ||\n (observedProps.Description ?? undefined) !== (news.description ?? undefined);\n if (drift) {\n // Computed outside the retry so replays of the same submission\n // reuse one idempotency token.\n const clientToken = yield* Effect.sync(() => crypto.randomUUID());\n yield* translate\n .updateParallelData({\n Name: name,\n ClientToken: clientToken,\n Description: news.description,\n ParallelDataConfig: {\n S3Uri: news.s3Uri,\n Format: news.format,\n },\n })\n .pipe(\n // A concurrent import may still be settling — retry bounded.\n Effect.retry({\n while: (e): boolean =>\n e._tag === \"ConcurrentModificationException\" ||\n e._tag === \"ConflictException\",\n schedule: Schedule.spaced(\"10 seconds\"),\n times: 8,\n }),\n );\n }\n }\n\n yield* session.note(name);\n const props = yield* waitForStable(name);\n yield* syncTranslateTags(props.Arn!, desiredTags);\n return toAttributes(props);\n }),\n\n delete: Effect.fn(function* ({ output }) {\n // Idempotent — the parallel data may already be gone. A delete\n // during CREATING/UPDATING conflicts; retry bounded.\n yield* translate.deleteParallelData({ Name: output.parallelDataName }).pipe(\n Effect.retry({\n while: (e): boolean => e._tag === \"ConcurrentModificationException\",\n schedule: Schedule.spaced(\"10 seconds\"),\n times: 8,\n }),\n Effect.catchTag(\"ResourceNotFoundException\", () => Effect.void),\n );\n // Deletion is asynchronous (DELETING) — wait until gone so a\n // replacement can immediately reuse the name.\n yield* translate.getParallelData({ Name: output.parallelDataName }).pipe(\n Effect.catchTag(\"ResourceNotFoundException\", () => Effect.undefined),\n Effect.repeat({\n schedule: Schedule.spaced(\"5 seconds\"),\n until: (r): boolean => r === undefined,\n times: 36,\n }),\n );\n }),\n\n list: () =>\n translate.listParallelData.pages({}).pipe(\n Stream.runCollect,\n Effect.map((chunk) =>\n Array.from(chunk)\n .flatMap((page) => page.ParallelDataPropertiesList ?? [])\n .filter((props) => props.Name && props.Arn)\n .map(toAttributes),\n ),\n ),\n };\n }),\n );\n","preview":null,"history":null,"version":"0b08ced148ad3954a92880653fcb1948cbf412ca"},"/_session?shell=1":null}