import * as kinesis from "@distilled.cloud/aws/kinesis";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import type { Input } from "alchemy/Input";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import type { Providers } from "../Providers.ts";
import type { StreamArn } from "./Stream.ts";
export class StreamConsumerUnexpectedResponse extends Data.TaggedError(
"StreamConsumerUnexpectedResponse",
)<{
readonly message: string;
readonly cause?: unknown;
}> {}
export type ConsumerName = string;
export type ConsumerArn = string;
export type ConsumerStatus = "CREATING" | "DELETING" | "ACTIVE";
export interface StreamConsumerProps {
/**
* ARN of the stream that owns the consumer.
*/
streamArn: Input<StreamArn>;
/**
* Name of the stream consumer.
* @default ${app}-${stage}-${id}
*/
consumerName?: string;
/**
* Tags to associate with the consumer.
*/
tags?: Record<string, string>;
}
export interface StreamConsumer extends Resource<
"AWS.Kinesis.StreamConsumer",
StreamConsumerProps,
{
/**
* The consumer's physical name.
*/
consumerName: ConsumerName;
/**
* ARN of the registered consumer.
*/
consumerArn: ConsumerArn;
/**
* Current lifecycle status of the consumer.
*/
consumerStatus: ConsumerStatus;
/**
* ARN of the stream the consumer is registered on.
*/
streamArn: StreamArn;
/**
* When the consumer was registered.
*/
consumerCreationTimestamp: Date;
/**
* Current tags reported for the consumer.
*/
tags: Record<string, string>;
},
never,
Providers
> {}
/**
* A registered Kinesis enhanced fan-out consumer.
*
* `StreamConsumer` is the canonical lifecycle resource for
* `RegisterStreamConsumer` / `DeregisterStreamConsumer`.
* ### Creating Consumers
* **Example:** Register a Consumer
* ```typescript
* const consumer = yield* StreamConsumer("AnalyticsConsumer", {
* streamArn: stream.streamArn,
* });
* ```
*
* @resource
*/
export const StreamConsumer = Resource<StreamConsumer>("AWS.Kinesis.StreamConsumer");
const createConsumerName = (
id: string,
props: {
consumerName?: string | undefined;
},
) =>
Effect.gen(function* () {
if (props.consumerName) {
return props.consumerName;
}
return yield* createPhysicalName({
id,
maxLength: 128,
});
});
const toTagRecord = (tags: Array<{ Key: string; Value?: string }> | undefined) =>
Object.fromEntries(
(tags ?? [])
.filter((tag): tag is { Key: string; Value: string } => typeof tag.Value === "string")
.map((tag) => [tag.Key, tag.Value]),
);
const toAttrs = ({
description,
tags,
}: {
description: kinesis.ConsumerDescription;
tags: Record<string, string>;
}): StreamConsumer["Attributes"] => ({
consumerName: description.ConsumerName,
consumerArn: description.ConsumerARN,
consumerStatus: description.ConsumerStatus as ConsumerStatus,
streamArn: description.StreamARN as StreamArn,
consumerCreationTimestamp: description.ConsumerCreationTimestamp,
tags,
});
const readConsumer = Effect.fn(function* ({
streamArn,
consumerName,
consumerArn,
}: {
streamArn?: string;
consumerName?: string;
consumerArn?: string;
}) {
const response = yield* kinesis
.describeStreamConsumer({
StreamARN: streamArn,
ConsumerName: consumerName,
ConsumerARN: consumerArn,
})
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
if (!response) {
return undefined;
}
const description = response.ConsumerDescription;
const tagsResponse = yield* kinesis.listTagsForResource({
ResourceARN: description.ConsumerARN,
});
return toAttrs({
description,
tags: toTagRecord(tagsResponse.Tags),
});
});
// Kinesis tells us "in use" via ResourceInUseException, but the consumer
// registry is eventually consistent — describeStreamConsumer can briefly
// return ResourceNotFoundException for a consumer the registry just
// confirmed exists. Poll up to ~10s before giving up.
class ConsumerRegistryNotConsistent extends Data.TaggedError("ConsumerRegistryNotConsistent")<{
consumerName: string;
}> {}
const adoptExistingConsumer = Effect.fn(function* (streamArn: string, consumerName: string) {
return yield* Effect.gen(function* () {
const state = yield* readConsumer({
streamArn,
consumerName,
});
if (!state) {
return yield* new ConsumerRegistryNotConsistent({ consumerName });
}
return state;
}).pipe(
Effect.retry({
while: (e) => e._tag === "ConsumerRegistryNotConsistent",
schedule: Schedule.max([Schedule.exponential(250), Schedule.recurs(8)]),
}),
Effect.catchTag("ConsumerRegistryNotConsistent", () =>
Effect.fail(
new StreamConsumerUnexpectedResponse({
message:
`consumer ${consumerName} exists but could not be read after ` +
`~10s of registry-consistency retries`,
}),
),
),
);
});
const waitForConsumerStatus = (consumerArn: string, expectedStatus: ConsumerStatus) =>
Effect.gen(function* () {
yield* Effect.sleep("2 seconds");
const response = yield* kinesis.describeStreamConsumer({
ConsumerARN: consumerArn,
});
if (response.ConsumerDescription.ConsumerStatus !== expectedStatus) {
return yield* Effect.fail({ _tag: "ConsumerStatusNotReady" as const });
}
return response.ConsumerDescription;
}).pipe(
Effect.retry({
while: (e: { _tag: string }) =>
e._tag === "ConsumerStatusNotReady" || e._tag === "ParseError",
schedule: Schedule.max([Schedule.exponential(500), Schedule.recurs(60)]),
}),
);
const waitForConsumerDeleted = (consumerArn: string) =>
Effect.gen(function* () {
yield* kinesis.describeStreamConsumer({
ConsumerARN: consumerArn,
});
return yield* Effect.fail({ _tag: "ConsumerStillExists" as const });
}).pipe(
Effect.retry({
while: (e: { _tag: string }) => e._tag === "ConsumerStillExists" || e._tag === "ParseError",
schedule: Schedule.max([Schedule.exponential(500), Schedule.recurs(60)]),
}),
Effect.catchTag("ResourceNotFoundException", () => Effect.void),
);
export const StreamConsumerProvider = () =>
Provider.succeed(StreamConsumer, {
stables: ["consumerArn", "consumerName"],
read: Effect.fn(function* ({ id, olds, output }) {
const consumerName = output?.consumerName ?? (yield* createConsumerName(id, olds ?? {}));
const streamArn = output?.streamArn ?? olds?.streamArn;
// describeStreamConsumer rejects with InvalidArgumentException unless
// either consumerARN, or both streamARN + consumerName, are provided.
// If the engine probed for adoption before our upstream Stream was
// created we'll have neither — treat that as "doesn't exist".
if (typeof streamArn !== "string" && !output?.consumerArn) {
return undefined;
}
const state = yield* readConsumer({
streamArn: typeof streamArn === "string" ? streamArn : undefined,
consumerName,
consumerArn: output?.consumerArn,
});
if (!state) return undefined;
return (yield* hasAlchemyTags(id, state.tags as Tags)) ? state : Unowned(state);
}),
diff: Effect.fn(function* ({ id, news, olds }) {
if (!isResolved(news)) return;
const oldConsumerName = yield* createConsumerName(id, olds);
const newConsumerName = yield* createConsumerName(id, news);
if (oldConsumerName !== newConsumerName) {
return { action: "replace" } as const;
}
if (
typeof news.streamArn === "string" &&
typeof olds.streamArn === "string" &&
news.streamArn !== olds.streamArn
) {
return { action: "replace" } as const;
}
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
const consumerName = output?.consumerName ?? (yield* createConsumerName(id, news));
const streamArn = news.streamArn as string;
const internalTags = yield* createInternalTags(id);
const desiredTags = { ...internalTags, ...news.tags };
// Observe — fetch live cloud state. `output` is treated as a cache
// for the consumer ARN; the consumer's actual existence and tags are
// fetched fresh so the reconciler converges regardless of drift,
// adoption, or a partially-completed prior run.
let state = yield* readConsumer({
streamArn,
consumerName,
consumerArn: output?.consumerArn,
});
// Ensure — register the consumer if it's missing. Tolerate
// `ResourceInUseException` as a race with a peer reconciler: a
// brief registry-consistency wait, then re-read and continue the
// sync path.
if (state === undefined) {
yield* kinesis
.registerStreamConsumer({
StreamARN: streamArn,
ConsumerName: consumerName,
Tags: desiredTags,
})
.pipe(
// The upstream Stream provider waits for ACTIVE via
// describeStreamSummary, but the consumer-registry control plane
// is eventually consistent against that: registerStreamConsumer
// can briefly 404 ("Stream ... not found") for a stream that
// describeStreamSummary already reports ACTIVE. Ride out that
// propagation window before giving up.
Effect.retry({
while: (e) => e._tag === "ResourceNotFoundException",
schedule: Schedule.max([Schedule.exponential(500), Schedule.recurs(8)]),
}),
Effect.asVoid,
Effect.catchTag("ResourceInUseException", () =>
adoptExistingConsumer(streamArn, consumerName).pipe(Effect.asVoid),
),
);
state = yield* adoptExistingConsumer(streamArn, consumerName);
yield* waitForConsumerStatus(state.consumerArn, "ACTIVE");
state = yield* readConsumer({
consumerArn: state.consumerArn,
streamArn,
consumerName,
});
if (state === undefined) {
return yield* new StreamConsumerUnexpectedResponse({
message: `failed to read created consumer ${consumerName}`,
});
}
}
// Sync tags — diff observed cloud tags against desired. Adoption
// may bring us a consumer that already has its own tag set; diffing
// against `state.tags` (fetched fresh) lets the reconciler converge
// ownership without fighting whatever was there before.
const { removed, upsert } = diffTags(state.tags, desiredTags);
if (removed.length > 0) {
yield* kinesis.untagResource({
ResourceARN: state.consumerArn,
TagKeys: removed,
});
}
if (upsert.length > 0) {
const tagsToAdd: Record<string, string> = {};
for (const { Key, Value } of upsert) {
tagsToAdd[Key] = Value;
}
yield* kinesis.tagResource({
ResourceARN: state.consumerArn,
Tags: tagsToAdd,
});
}
// Re-read final state so the returned attributes reflect what's
// actually in the cloud after all sync steps.
const final = yield* readConsumer({
consumerArn: state.consumerArn,
});
if (!final) {
return yield* new StreamConsumerUnexpectedResponse({
message: `failed to read reconciled consumer ${consumerName}`,
});
}
yield* session.note(final.consumerArn);
return final;
}),
delete: Effect.fn(function* ({ output }) {
yield* kinesis
.deregisterStreamConsumer({
ConsumerARN: output.consumerArn,
})
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
yield* waitForConsumerDeleted(output.consumerArn);
}),
// Consumers are stream-scoped: listStreamConsumers requires a StreamARN.
// Fan out — enumerate every stream in the account/region, then list the
// consumers registered on each stream, then hydrate tags per consumer.
list: Effect.fn(function* () {
const streamArns = yield* kinesis.listStreams.pages({}).pipe(
Stream.runCollect,
Effect.map((chunk) =>
Array.from(chunk).flatMap((page) =>
(page.StreamSummaries ?? [])
.map((summary) => summary.StreamARN)
.filter((arn): arn is string => typeof arn === "string"),
),
),
);
const consumers = yield* Effect.forEach(
streamArns,
(streamArn) =>
kinesis.listStreamConsumers.pages({ StreamARN: streamArn }).pipe(
Stream.runCollect,
Effect.map((chunk) =>
Array.from(chunk).flatMap((page) =>
(page.Consumers ?? []).map((consumer) => ({
streamArn,
consumer,
})),
),
),
// A stream may be deleted mid-enumeration; skip it.
Effect.catchTag("ResourceNotFoundException", () =>
Effect.succeed([] as { streamArn: string; consumer: kinesis.Consumer }[]),
),
),
{ concurrency: 10 },
).pipe(Effect.map((rows) => rows.flat()));
const hydrated = yield* Effect.forEach(
consumers,
({ streamArn, consumer }) =>
kinesis.listTagsForResource({ ResourceARN: consumer.ConsumerARN }).pipe(
Effect.map((tagsResponse): StreamConsumer["Attributes"] => ({
consumerName: consumer.ConsumerName,
consumerArn: consumer.ConsumerARN,
consumerStatus: consumer.ConsumerStatus as ConsumerStatus,
streamArn: streamArn as StreamArn,
consumerCreationTimestamp: consumer.ConsumerCreationTimestamp,
tags: toTagRecord(tagsResponse.Tags),
})),
// A consumer may be deregistered between list and tag fetch.
Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
),
{ concurrency: 10 },
);
return hydrated.filter((row): row is StreamConsumer["Attributes"] => row !== undefined);
}),
});