import * as sagemaker from "@distilled.cloud/aws/sagemaker";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as EffectStream from "effect/Stream";
import type { Providers } from "../Providers.ts";
export class ComputeQuotaInvalidProps extends Data.TaggedError("ComputeQuotaInvalidProps")<{
readonly message: string;
readonly cause?: unknown;
}> {}
export class ComputeQuotaUnexpectedResponse extends Data.TaggedError(
"ComputeQuotaUnexpectedResponse",
)<{
readonly message: string;
readonly cause?: unknown;
}> {}
export interface ComputeQuotaProps {
/**
* Name of the compute allocation. Maximum 63 characters, alphanumeric and
* hyphens.
* @default ${app}-${stage}-${id}
*/
name?: string;
/**
* ARN of the EKS-orchestrated HyperPod cluster the allocation applies to.
* Changing the cluster replaces the allocation.
*/
clusterArn: string;
/**
* The team the quota is allocated to (its name maps to a Kubernetes
* namespace `hyperpod-ns-<team-name>`) and its fair-share weight.
*/
computeQuotaTarget: sagemaker.ComputeQuotaTarget;
/**
* The allocation itself: instance-type quotas, the resource-sharing
* strategy for idle compute, and whether the team's own tasks can preempt
* each other.
*/
computeQuotaConfig?: sagemaker.ComputeQuotaConfig;
/**
* Whether the quota is enforced.
* @default "Enabled"
*/
activationState?: sagemaker.ActivationState;
/**
* A description of the compute allocation.
*/
description?: string;
/**
* Tags to associate with the compute allocation. Merged with internal
* Alchemy tags.
*/
tags?: Record<string, string>;
}
/**
* Kueue label selecting the task-governance queue. Set it on a
* `Kubernetes.Job` / `Kubernetes.Deployment`'s `labels` with the quota's
* `queueName` attribute to submit the workload through HyperPod task
* governance.
*/
export const KUEUE_QUEUE_NAME_LABEL = "kueue.x-k8s.io/queue-name";
/**
* Kueue label selecting the task-governance priority class. Its value is
* `<PriorityClass name>-priority` for a class declared on the cluster's
* `AWS.SageMaker.ClusterSchedulerConfig`.
*/
export const KUEUE_PRIORITY_CLASS_LABEL = "kueue.x-k8s.io/priority-class";
export interface ComputeQuota extends Resource<
"AWS.SageMaker.ComputeQuota",
ComputeQuotaProps,
{
/**
* The compute allocation's ID.
*/
computeQuotaId: string;
/**
* ARN of the compute allocation.
*/
computeQuotaArn: string;
/**
* The compute allocation's name.
*/
name: string;
/**
* ARN of the HyperPod cluster the allocation applies to.
*/
clusterArn: string;
/**
* The allocation's current version (incremented on every update).
*/
computeQuotaVersion: number;
/**
* The team the quota is allocated to. Task governance materializes the
* `hyperpod-ns-<teamName>` namespace and its Kueue LocalQueue from it.
*/
teamName: string;
/**
* The Kubernetes namespace task governance materializes for the team
* (`hyperpod-ns-<teamName>`). Pass it as a governed
* `Kubernetes.Job` / `Kubernetes.Deployment`'s `namespace`.
*/
namespace: string;
/**
* The team's Kueue LocalQueue name
* (`hyperpod-ns-<teamName>-localqueue`). Set it on a governed
* workload's `labels` under {@link KUEUE_QUEUE_NAME_LABEL}.
*/
queueName: string;
},
never,
Providers
> {}
/**
* A SageMaker HyperPod compute allocation (task governance) — reserves
* instance capacity on an EKS-orchestrated HyperPod cluster for a team,
* with fair-share weights and borrow/lend rules for idle compute.
* ### Creating Compute Allocations
* **Example:** Team Quota with Borrowing
* ```typescript
* import * as AWS from "@alchemy.run/aws";
*
* const quota = yield* AWS.SageMaker.ComputeQuota("ResearchQuota", {
* clusterArn: hyperpod.clusterArn,
* computeQuotaTarget: { TeamName: "research", FairShareWeight: 10 },
* computeQuotaConfig: {
* ComputeQuotaResources: [{ InstanceType: "ml.g5.xlarge", Count: 2 }],
* ResourceSharingConfig: { Strategy: "Lend", BorrowLimit: 50 },
* },
* });
* ```
*
* @resource
*/
export const ComputeQuota = Resource<ComputeQuota>("AWS.SageMaker.ComputeQuota");
const createQuotaName = (id: string, props: { name?: string | undefined }) =>
props.name ? Effect.succeed(props.name) : createPhysicalName({ id, maxLength: 63 });
const describeQuotaOrUndefined = (quotaId: string) =>
sagemaker
.describeComputeQuota({ ComputeQuotaId: quotaId })
.pipe(Effect.catchTag("ResourceNotFound", () => Effect.undefined));
/**
* Look a compute allocation up by exact name — used when state was lost
* (read without output) or a create raced.
*/
const findQuotaByName = Effect.fn(function* (name: string) {
const summaries = yield* sagemaker.listComputeQuotas.pages({ NameContains: name }).pipe(
EffectStream.runCollect,
Effect.map((chunk) => Array.from(chunk).flatMap((page) => page.ComputeQuotaSummaries ?? [])),
);
return summaries.find((s) => s.Name === name && s.Status !== "Deleted");
});
const fetchQuotaTags = Effect.fn(function* (arn: string) {
const response = yield* sagemaker
.listTags({ ResourceArn: arn })
.pipe(Effect.catchTag("AccessDeniedException", () => Effect.undefined));
return Object.fromEntries(
(response?.Tags ?? []).flatMap((tag) =>
tag.Key !== undefined ? [[tag.Key, tag.Value ?? ""]] : [],
),
);
});
const toAttrs = (described: sagemaker.DescribeComputeQuotaResponse): ComputeQuota["Attributes"] => {
const teamName = described.ComputeQuotaTarget?.TeamName ?? "";
return {
computeQuotaId: described.ComputeQuotaId,
computeQuotaArn: described.ComputeQuotaArn,
name: described.Name,
clusterArn: described.ClusterArn ?? "",
computeQuotaVersion: described.ComputeQuotaVersion,
teamName,
// Task-governance conventions: the quota materializes the team's
// namespace and Kueue LocalQueue under these derived names.
namespace: `hyperpod-ns-${teamName}`,
queueName: `hyperpod-ns-${teamName}-localqueue`,
};
};
/**
* The compute allocation is still transitioning toward the awaited state —
* retried by the bounded wait schedule.
*/
class ComputeQuotaNotReady extends Data.TaggedError("ComputeQuotaNotReady")<{
readonly quotaId: string;
readonly status: string | undefined;
}> {}
/**
* The compute allocation converged to a terminal failed status.
*/
export class ComputeQuotaFailed extends Data.TaggedError("ComputeQuotaFailed")<{
readonly quotaId: string;
readonly status: string | undefined;
readonly message: string | undefined;
}> {}
const FAILED_STATUSES: string[] = [
"CreateFailed",
"CreateRollbackFailed",
"UpdateFailed",
"UpdateRollbackFailed",
"DeleteFailed",
"DeleteRollbackFailed",
];
// Explicitly-typed retry wrapper — an inline `Effect.retry` in provider
// lifecycle code leaks `Retry.Return`'s conditional type into declaration
// emit and widens the provider layer to `unknown` for every consumer of
// `AWS.providers()`.
const retryWhileNotReady = <A, E extends { readonly _tag: string }, R>(
self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
Effect.retry(self, {
while: (e) => e._tag === "ComputeQuotaNotReady",
schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(60)]),
});
const waitForQuota = (quotaId: string, target: "Ready" | "Gone") =>
retryWhileNotReady(
Effect.gen(function* () {
const described = yield* describeQuotaOrUndefined(quotaId);
if (target === "Gone") {
if (described === undefined || described.Status === "Deleted") return;
if (FAILED_STATUSES.includes(described.Status)) {
return yield* new ComputeQuotaFailed({
quotaId,
status: described.Status,
message: described.FailureReason,
});
}
return yield* new ComputeQuotaNotReady({ quotaId, status: described.Status });
}
if (described?.Status === "Created" || described?.Status === "Updated") {
return;
}
if (described !== undefined && FAILED_STATUSES.includes(described.Status)) {
return yield* new ComputeQuotaFailed({
quotaId,
status: described.Status,
message: described.FailureReason,
});
}
return yield* new ComputeQuotaNotReady({ quotaId, status: described?.Status });
}),
);
export const ComputeQuotaProvider = () =>
Provider.effect(
ComputeQuota,
Effect.gen(function* () {
return {
stables: ["computeQuotaId", "computeQuotaArn", "name", "clusterArn"],
list: () =>
Effect.gen(function* () {
const summaries = yield* sagemaker.listComputeQuotas.pages({}).pipe(
EffectStream.runCollect,
Effect.map((chunk) =>
Array.from(chunk).flatMap((page) => page.ComputeQuotaSummaries ?? []),
),
);
return summaries.flatMap((s) => {
if (s.ComputeQuotaId === undefined || s.Status === "Deleted") {
return [];
}
const teamName = s.ComputeQuotaTarget?.TeamName ?? "";
return [
{
computeQuotaId: s.ComputeQuotaId,
computeQuotaArn: s.ComputeQuotaArn ?? "",
name: s.Name ?? "",
clusterArn: s.ClusterArn ?? "",
computeQuotaVersion: s.ComputeQuotaVersion ?? 1,
teamName,
namespace: `hyperpod-ns-${teamName}`,
queueName: `hyperpod-ns-${teamName}-localqueue`,
},
];
});
}),
read: Effect.fn(function* ({ id, olds, output }) {
const quotaId =
output?.computeQuotaId ??
(yield* findQuotaByName(yield* createQuotaName(id, olds ?? {})))?.ComputeQuotaId;
if (quotaId === undefined) return undefined;
const described = yield* describeQuotaOrUndefined(quotaId);
if (!described || described.Status === "Deleting" || described.Status === "Deleted") {
return undefined;
}
const attrs = toAttrs(described);
const tags = yield* fetchQuotaTags(attrs.computeQuotaArn);
return (yield* hasAlchemyTags(id, tags as Tags)) ? attrs : Unowned(attrs);
}),
diff: Effect.fn(function* ({ id, news, olds }) {
if (!isResolved(news)) return;
if (olds === undefined) return;
const oldName = yield* createQuotaName(id, olds);
const newName = yield* createQuotaName(id, news);
// The name and target cluster are fixed at creation.
if (oldName !== newName || olds.clusterArn !== news.clusterArn) {
return { action: "replace" } as const;
}
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
if (!news) {
return yield* new ComputeQuotaInvalidProps({
message: "SageMaker ComputeQuota requires props",
});
}
const name = yield* createQuotaName(id, news);
const internalTags = yield* createInternalTags(id);
const desiredTags = { ...internalTags, ...news.tags };
// Observe — by cached id first, then by name (lost state or race).
let quotaId = output?.computeQuotaId;
let described =
quotaId !== undefined ? yield* describeQuotaOrUndefined(quotaId) : undefined;
if (described === undefined) {
const found = yield* findQuotaByName(name);
described =
found?.ComputeQuotaId !== undefined
? yield* describeQuotaOrUndefined(found.ComputeQuotaId)
: undefined;
}
// Ensure — create if missing; a Conflict means a concurrent
// create won the race, so re-observe by name.
if (described === undefined) {
const created = yield* sagemaker
.createComputeQuota({
Name: name,
ClusterArn: news.clusterArn,
ComputeQuotaConfig: news.computeQuotaConfig,
ComputeQuotaTarget: news.computeQuotaTarget,
ActivationState: news.activationState,
Description: news.description,
Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
Key,
Value,
})),
})
.pipe(Effect.catchTag("ConflictException", () => Effect.undefined));
quotaId = created?.ComputeQuotaId ?? (yield* findQuotaByName(name))?.ComputeQuotaId;
if (quotaId === undefined) {
return yield* new ComputeQuotaUnexpectedResponse({
message: `failed to create compute quota ${name}`,
});
}
yield* session.note(`Creating compute quota ${name}...`);
yield* waitForQuota(quotaId, "Ready");
} else {
quotaId = described.ComputeQuotaId;
// Wait out any in-flight transition before diffing.
yield* waitForQuota(quotaId, "Ready");
described = yield* describeQuotaOrUndefined(quotaId);
// Sync — diff observed allocation against desired.
if (
described !== undefined &&
(JSON.stringify(described.ComputeQuotaConfig) !==
JSON.stringify(news.computeQuotaConfig) ||
JSON.stringify(described.ComputeQuotaTarget) !==
JSON.stringify(news.computeQuotaTarget) ||
(news.activationState !== undefined &&
described.ActivationState !== news.activationState) ||
(described.Description ?? undefined) !== (news.description ?? undefined))
) {
yield* sagemaker.updateComputeQuota({
ComputeQuotaId: quotaId,
TargetVersion: described.ComputeQuotaVersion,
ComputeQuotaConfig: news.computeQuotaConfig,
ComputeQuotaTarget: news.computeQuotaTarget,
ActivationState: news.activationState,
Description: news.description,
});
yield* session.note(`Updating compute quota ${name}...`);
yield* waitForQuota(quotaId, "Ready");
}
}
described = yield* describeQuotaOrUndefined(quotaId);
if (described === undefined) {
return yield* new ComputeQuotaUnexpectedResponse({
message: `failed to read reconciled compute quota ${name}`,
});
}
const attrs = toAttrs(described);
// Sync tags — diff against OBSERVED cloud tags.
const currentTags = yield* fetchQuotaTags(attrs.computeQuotaArn);
const { removed, upsert } = diffTags(currentTags, desiredTags);
if (removed.length > 0) {
yield* sagemaker.deleteTags({
ResourceArn: attrs.computeQuotaArn,
TagKeys: removed,
});
}
if (upsert.length > 0) {
yield* sagemaker.addTags({
ResourceArn: attrs.computeQuotaArn,
Tags: upsert.map(({ Key, Value }) => ({ Key, Value })),
});
}
yield* session.note(attrs.computeQuotaArn);
return attrs;
}),
delete: Effect.fn(function* ({ output }) {
yield* sagemaker
.deleteComputeQuota({ ComputeQuotaId: output.computeQuotaId })
.pipe(Effect.catchTag("ResourceNotFound", () => Effect.void));
yield* waitForQuota(output.computeQuotaId, "Gone");
}),
};
}),
);