import * as finspace from "@distilled.cloud/aws/finspace";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags } from "alchemy/Tags";
import { toWireSeconds } from "alchemy/Util/Duration";
import * as Data from "effect/Data";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import type { Providers } from "../Providers.ts";
export class KxClusterInvalidProps extends Data.TaggedError("KxClusterInvalidProps")<{
readonly message: string;
readonly cause?: unknown;
}> {}
export class KxClusterUnexpectedResponse extends Data.TaggedError("KxClusterUnexpectedResponse")<{
readonly message: string;
readonly cause?: unknown;
}> {}
export type KxClusterStatus = finspace.KxClusterStatus;
export type KxClusterType = finspace.KxClusterType;
export type KxAzMode = finspace.KxAzMode;
export type VpcConfiguration = finspace.VpcConfiguration;
export type CapacityConfiguration = finspace.CapacityConfiguration;
export type CodeConfiguration = finspace.CodeConfiguration;
export type KxCommandLineArgument = finspace.KxCommandLineArgument;
export type KxDatabaseConfiguration = finspace.KxDatabaseConfiguration;
export type KxCacheStorageConfiguration = finspace.KxCacheStorageConfiguration;
export type KxSavedownStorageConfiguration = finspace.KxSavedownStorageConfiguration;
export type KxScalingGroupConfiguration = finspace.KxScalingGroupConfiguration;
export type TickerplantLogConfiguration = finspace.TickerplantLogConfiguration;
/**
* Auto-scaling policy for a dedicated {@link KxCluster}.
*/
export interface AutoScalingConfiguration {
/** Lowest number of nodes to scale in to. */
minNodeCount?: number;
/** Highest number of nodes to scale out to. */
maxNodeCount?: number;
/** The metric the auto-scaling policy tracks. */
autoScalingMetric?: finspace.AutoScalingMetric;
/** The desired value of the chosen metric. */
metricTarget?: number;
/**
* The cooldown after a scale-in event before another scaling event —
* e.g. `"5 minutes"`. Sent to AWS as whole seconds.
*/
scaleInCooldown?: Duration.Input;
/**
* The cooldown after a scale-out event before another scaling event —
* e.g. `"5 minutes"`. Sent to AWS as whole seconds.
*/
scaleOutCooldown?: Duration.Input;
}
/** Convert the alchemy-facing auto-scaling config to the wire shape (seconds). */
const toWireAutoScaling = (
config: AutoScalingConfiguration,
): finspace.AutoScalingConfiguration => ({
minNodeCount: config.minNodeCount,
maxNodeCount: config.maxNodeCount,
autoScalingMetric: config.autoScalingMetric,
metricTarget: config.metricTarget,
scaleInCooldownSeconds: toWireSeconds(config.scaleInCooldown),
scaleOutCooldownSeconds: toWireSeconds(config.scaleOutCooldown),
});
export interface KxClusterProps {
/**
* Identifier of the kdb environment the cluster runs in. Changing it
* replaces the cluster.
*/
environmentId: string;
/**
* Name of the kdb cluster. Changing it replaces the cluster.
* @default ${app}-${id}-${stage}-${suffix}
*/
clusterName?: string;
/**
* The type of cluster — `HDB`, `RDB`, `GATEWAY`, `GP` or `TICKERPLANT`.
* Changing it replaces the cluster.
*/
clusterType: KxClusterType;
/**
* The kdb release the cluster runs (e.g. `"1.0"`). Changing it replaces
* the cluster.
*/
releaseLabel: string;
/**
* VPC the cluster network interfaces are placed in. Changing it replaces
* the cluster.
*/
vpcConfiguration: VpcConfiguration;
/**
* Availability-zone placement mode — `SINGLE` or `MULTI`. Changing it
* replaces the cluster.
*/
azMode: KxAzMode;
/**
* The availability zone id to place the cluster in (required when
* `azMode` is `SINGLE`). Changing it replaces the cluster.
*/
availabilityZoneId?: string;
/**
* Dedicated node capacity for the cluster. Mutually exclusive with
* `scalingGroupConfiguration`. Changing it replaces the cluster.
*/
capacityConfiguration?: CapacityConfiguration;
/**
* Placement on a shared scaling group instead of dedicated capacity.
* Changing it replaces the cluster.
*/
scalingGroupConfiguration?: KxScalingGroupConfiguration;
/**
* Auto-scaling policy for a dedicated cluster. Changing it replaces the
* cluster.
*/
autoScalingConfiguration?: AutoScalingConfiguration;
/**
* Temporary savedown storage for an `RDB` cluster. Changing it replaces
* the cluster.
*/
savedownStorageConfiguration?: KxSavedownStorageConfiguration;
/**
* Databases to mount on the cluster, with optional cache configuration.
*/
databases?: KxDatabaseConfiguration[];
/**
* Cache storage (e.g. `CACHE_1000`) sized for the mounted databases.
* Changing it replaces the cluster.
*/
cacheStorageConfigurations?: KxCacheStorageConfiguration[];
/**
* Tickerplant log volumes for a `TICKERPLANT` cluster. Changing it
* replaces the cluster.
*/
tickerplantLogConfiguration?: TickerplantLogConfiguration;
/**
* A description of the cluster.
*/
description?: string;
/**
* The S3 location of the q code to deploy on the cluster.
*/
code?: CodeConfiguration;
/**
* Path (relative to the code root) of the q script run at cluster start.
*/
initializationScript?: string;
/**
* Command-line arguments passed to the kdb process.
*/
commandLineArguments?: KxCommandLineArgument[];
/**
* ARN of the IAM execution role the cluster assumes to access AWS
* resources.
*/
executionRole?: string;
/**
* Tags to associate with the cluster (applied at creation).
*/
tags?: Record<string, string>;
}
export interface KxCluster extends Resource<
"AWS.FinSpace.KxCluster",
KxClusterProps,
{
/**
* Identifier of the kdb environment the cluster runs in.
*/
environmentId: string;
/**
* The cluster's name.
*/
clusterName: string;
/**
* Current lifecycle status of the cluster.
*/
status: KxClusterStatus | undefined;
/**
* The cluster's type.
*/
clusterType: KxClusterType | undefined;
/**
* The kdb release the cluster runs.
*/
releaseLabel: string | undefined;
/**
* Availability-zone placement mode.
*/
azMode: KxAzMode | undefined;
},
never,
Providers
> {}
/**
* A kdb cluster inside an Amazon FinSpace Managed kdb environment — the
* compute that mounts kdb databases and serves q queries.
*
* :::caution
* Cluster provisioning is slow (tens of minutes) and bills per node-hour
* while it exists. Live lifecycle tests are gated behind
* `AWS_TEST_FINSPACE=1`.
* :::
* ### Creating kdb Clusters
* **Example:** HDB Cluster on Dedicated Capacity
* ```typescript
* import * as AWS from "@alchemy.run/aws";
*
* const cluster = yield* AWS.FinSpace.KxCluster("Hdb", {
* environmentId: env.environmentId,
* clusterType: "HDB",
* releaseLabel: "1.0",
* azMode: "SINGLE",
* availabilityZoneId: "use1-az1",
* capacityConfiguration: { nodeType: "kx.s.large", nodeCount: 1 },
* vpcConfiguration: {
* vpcId: vpc.vpcId,
* securityGroupIds: [sg.securityGroupId],
* subnetIds: [subnet.subnetId],
* ipAddressType: "IP_V4",
* },
* databases: [{ databaseName: db.databaseName }],
* });
* ```
*
* @resource
*/
export const KxCluster = Resource<KxCluster>("AWS.FinSpace.KxCluster");
const createClusterName = (id: string, props: { clusterName?: string | undefined }) =>
props.clusterName ? Effect.succeed(props.clusterName) : createPhysicalName({ id, maxLength: 63 });
const isGone = (status: KxClusterStatus | undefined) =>
status === "DELETED" || status === "DELETING";
const readCluster = Effect.fn(function* (environmentId: string, clusterName: string) {
const response = yield* finspace
.getKxCluster({ environmentId, clusterName })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
if (!response || isGone(response.status)) return undefined;
return response;
});
const toAttributes = (
cluster: finspace.GetKxClusterResponse,
environmentId: string,
clusterName: string,
): KxCluster["Attributes"] => ({
environmentId,
clusterName: cluster.clusterName ?? clusterName,
status: cluster.status,
clusterType: cluster.clusterType,
releaseLabel: cluster.releaseLabel,
azMode: cluster.azMode,
});
/**
* A cluster still transitioning toward the awaited status — retried by the
* bounded schedule in {@link waitForClusterStatus}.
*/
class KxClusterNotReady extends Data.TaggedError("KxClusterNotReady")<{
readonly clusterName: string;
readonly status: string | undefined;
}> {}
/**
* A cluster whose asynchronous provisioning converged to a terminal failure
* status (`CREATE_FAILED` / `DELETE_FAILED`).
*/
export class KxClusterProvisioningFailed extends Data.TaggedError("KxClusterProvisioningFailed")<{
readonly clusterName: string;
readonly status: string | undefined;
readonly statusReason: string | undefined;
}> {}
// Explicitly-typed retry wrapper — an inline `Effect.retry` in provider
// lifecycle code leaks `Retry.Return`'s conditional type into declaration
// emit and widens the provider layer to `unknown` for every consumer of
// `AWS.providers()`.
const retryWhileNotReady = <A, E extends { readonly _tag: string }, R>(
self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
Effect.retry(self, {
while: (e) => e._tag === "KxClusterNotReady",
// Cluster provisioning is slow (tens of minutes); poll every 20s up to
// ~40 min.
schedule: Schedule.max([Schedule.spaced("20 seconds"), Schedule.recurs(120)]),
});
const waitForClusterStatus = (
environmentId: string,
clusterName: string,
target: "RUNNING" | "DELETED",
) =>
retryWhileNotReady(
Effect.gen(function* () {
const response = yield* finspace
.getKxCluster({ environmentId, clusterName })
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
const status = response?.status;
if (target === "DELETED") {
if (response === undefined || status === "DELETED") return;
if (status === "DELETE_FAILED") {
return yield* new KxClusterProvisioningFailed({
clusterName,
status,
statusReason: response.statusReason,
});
}
return yield* new KxClusterNotReady({ clusterName, status });
}
if (status === "RUNNING") return;
if (status === "CREATE_FAILED") {
return yield* new KxClusterProvisioningFailed({
clusterName,
status,
statusReason: response?.statusReason,
});
}
// response === undefined right after create is eventual consistency —
// keep polling on the bounded schedule.
return yield* new KxClusterNotReady({ clusterName, status });
}),
);
const sameJson = (a: unknown, b: unknown) =>
JSON.stringify(a ?? null) === JSON.stringify(b ?? null);
export const KxClusterProvider = () =>
Provider.effect(
KxCluster,
Effect.gen(function* () {
return {
stables: ["environmentId", "clusterName"],
// Clusters are keyed by their parent kdb environment — there is no
// account-wide enumeration without an environment id.
list: () => Effect.succeed([]),
read: Effect.fn(function* ({ id, olds, output }) {
const environmentId = output?.environmentId ?? olds?.environmentId;
if (environmentId === undefined) return undefined;
const clusterName = output?.clusterName ?? (yield* createClusterName(id, olds ?? {}));
const cluster = yield* readCluster(environmentId, clusterName);
if (!cluster) return undefined;
// Clusters have no tag-read surface (no ARN in Get/List responses)
// — ownership cannot be verified, so a name match is owned.
return toAttributes(cluster, environmentId, clusterName);
}),
diff: Effect.fn(function* ({ id, news, olds }) {
if (!isResolved(news)) return;
if (olds === undefined) return;
const oldName = yield* createClusterName(id, olds);
const newName = yield* createClusterName(id, news);
// Everything except code/databases/description is fixed at
// creation.
if (
olds.environmentId !== news.environmentId ||
oldName !== newName ||
olds.clusterType !== news.clusterType ||
olds.releaseLabel !== news.releaseLabel ||
olds.azMode !== news.azMode ||
olds.availabilityZoneId !== news.availabilityZoneId ||
!sameJson(olds.vpcConfiguration, news.vpcConfiguration) ||
!sameJson(olds.capacityConfiguration, news.capacityConfiguration) ||
!sameJson(olds.scalingGroupConfiguration, news.scalingGroupConfiguration) ||
!sameJson(
olds.autoScalingConfiguration && toWireAutoScaling(olds.autoScalingConfiguration),
news.autoScalingConfiguration && toWireAutoScaling(news.autoScalingConfiguration),
) ||
!sameJson(olds.savedownStorageConfiguration, news.savedownStorageConfiguration) ||
!sameJson(olds.cacheStorageConfigurations, news.cacheStorageConfigurations) ||
!sameJson(olds.tickerplantLogConfiguration, news.tickerplantLogConfiguration) ||
olds.executionRole !== news.executionRole
) {
return { action: "replace" } as const;
}
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
if (!news) {
return yield* new KxClusterInvalidProps({
message: "FinSpace KxCluster requires props",
});
}
const environmentId = news.environmentId;
const clusterName = output?.clusterName ?? (yield* createClusterName(id, news));
const internalTags = yield* createInternalTags(id);
// One idempotency token per reconcile.
const clientToken = yield* Effect.sync(() => crypto.randomUUID());
// Observe — cloud state is authoritative.
let cluster = yield* readCluster(environmentId, clusterName);
// Ensure — create if missing (tolerating a Conflict race), then
// wait for RUNNING.
if (cluster === undefined) {
yield* finspace
.createKxCluster({
clientToken,
environmentId,
clusterName,
clusterType: news.clusterType,
releaseLabel: news.releaseLabel,
vpcConfiguration: news.vpcConfiguration,
azMode: news.azMode,
availabilityZoneId: news.availabilityZoneId,
capacityConfiguration: news.capacityConfiguration,
scalingGroupConfiguration: news.scalingGroupConfiguration,
autoScalingConfiguration:
news.autoScalingConfiguration && toWireAutoScaling(news.autoScalingConfiguration),
savedownStorageConfiguration: news.savedownStorageConfiguration,
databases: news.databases,
cacheStorageConfigurations: news.cacheStorageConfigurations,
tickerplantLogConfiguration: news.tickerplantLogConfiguration,
clusterDescription: news.description,
code: news.code,
initializationScript: news.initializationScript,
commandLineArguments: news.commandLineArguments,
executionRole: news.executionRole,
tags: { ...internalTags, ...news.tags },
})
.pipe(Effect.catchTag("ConflictException", () => Effect.undefined));
yield* session.note(`Creating kdb cluster ${clusterName}...`);
yield* waitForClusterStatus(environmentId, clusterName, "RUNNING");
cluster = yield* readCluster(environmentId, clusterName);
if (cluster === undefined) {
return yield* new KxClusterUnexpectedResponse({
message: `failed to read created kdb cluster ${clusterName}`,
});
}
}
// Sync code configuration — only when the caller manages code and
// the observed deployment drifted.
if (
news.code !== undefined &&
(!sameJson(news.code, cluster.code) ||
(news.initializationScript ?? "") !== (cluster.initializationScript ?? "") ||
!sameJson(news.commandLineArguments, cluster.commandLineArguments))
) {
yield* finspace.updateKxClusterCodeConfiguration({
environmentId,
clusterName,
clientToken,
code: news.code,
initializationScript: news.initializationScript,
commandLineArguments: news.commandLineArguments,
});
yield* session.note(`Updated code on kdb cluster ${clusterName}`);
yield* waitForClusterStatus(environmentId, clusterName, "RUNNING");
}
// Sync mounted databases — only when the caller manages them and
// the observed set drifted.
if (news.databases !== undefined && !sameJson(news.databases, cluster.databases)) {
yield* finspace.updateKxClusterDatabases({
environmentId,
clusterName,
clientToken,
databases: news.databases,
});
yield* session.note(`Updated databases on kdb cluster ${clusterName}`);
yield* waitForClusterStatus(environmentId, clusterName, "RUNNING");
}
yield* session.note(clusterName);
const final = yield* readCluster(environmentId, clusterName);
if (!final) {
return yield* new KxClusterUnexpectedResponse({
message: `failed to read reconciled kdb cluster ${clusterName}`,
});
}
return toAttributes(final, environmentId, clusterName);
}),
delete: Effect.fn(function* ({ output }) {
yield* finspace
.deleteKxCluster({
environmentId: output.environmentId,
clusterName: output.clusterName,
})
.pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
yield* waitForClusterStatus(output.environmentId, output.clusterName, "DELETED");
}),
};
}),
);