import { unownedUnlessPrevious } from "alchemy/AdoptPolicy";
import { deepEqual, isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
import * as Config from "effect/Config";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import * as ChildProcess from "effect/process/ChildProcess";
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
import * as Stream from "effect/Stream";
import { applyObject, connectCluster } from "./Client.ts";
import type { Connection, ContainerRegistry } from "./Connection.ts";
import { kindClusterConfig, kindConfigOf } from "./internal/kind.ts";
import type { Providers } from "./Providers.ts";
export interface LocalClusterProps extends KindClusterConfig {
/**
* Name of the kind cluster. Its kubeconfig context is `kind-<name>`.
* Changing it replaces the cluster.
* @default a name generated from the stack, stage, and logical ID
*/
name?: string;
/**
* Host port of the cluster's image registry, reachable from the deploying
* machine as `localhost:<registryPort>`. Give each local cluster its own
* port.
* @default 5001
*/
registryPort?: number;
/**
* The kind node image, which selects the Kubernetes version (e.g.
* `kindest/node:v1.33.1`). A node's own `image` overrides it. Changing it
* replaces the cluster.
* @default kind's default node image
*/
nodeImage?: string;
/**
* The kubeconfig file kind writes the cluster's context into. Changing it
* replaces the cluster.
* @default `$KUBECONFIG` or `~/.kube/config`
*/
kubeconfig?: string;
}
/**
* The fields of a [kind `Cluster` config](https://kind.sigs.k8s.io/docs/user/configuration/)
* (`kind.x-k8s.io/v1alpha4`), accepted directly on `LocalCluster`. kind
* can't reconfigure a running cluster, so changing any of them replaces the
* cluster.
*/
export interface KindClusterConfig {
/**
* The cluster's nodes.
* @default a single control-plane node
*/
nodes?: KindNode[];
/** Cluster networking. */
networking?: KindNetworking;
/** Kubernetes feature gates, e.g. `{ InPlacePodVerticalScaling: true }`. */
featureGates?: Record<string, boolean>;
/** API server `--runtime-config` entries, e.g. `{ "api/alpha": "false" }`. */
runtimeConfig?: Record<string, string>;
/** kubeadm config patches (YAML strategic-merge) applied to every node. */
kubeadmConfigPatches?: string[];
/** kubeadm config patches (RFC 6902 JSON patches) applied to every node. */
kubeadmConfigPatchesJSON6902?: KindJsonPatch[];
/**
* containerd config patches (TOML). Alchemy appends its own for the image
* registry.
*/
containerdConfigPatches?: string[];
/** containerd config patches (RFC 6902 JSON patches). */
containerdConfigPatchesJSON6902?: string[];
}
/** A node in a kind cluster. */
export interface KindNode {
/** Whether the node runs the control plane or only workloads. */
role: "control-plane" | "worker";
/** Node image for this node, overriding `nodeImage`. */
image?: string;
/** Kubernetes labels for the node. */
labels?: Record<string, string>;
/** Host ports forwarded to the node container. */
extraPortMappings?: KindPortMapping[];
/** Host paths mounted into the node container. */
extraMounts?: KindMount[];
/** kubeadm config patches (YAML strategic-merge) for this node. */
kubeadmConfigPatches?: string[];
/** kubeadm config patches (RFC 6902 JSON patches) for this node. */
kubeadmConfigPatchesJSON6902?: KindJsonPatch[];
}
/** A host port forwarded to a kind node container. */
export interface KindPortMapping {
/** Port on the node container, e.g. a NodePort or hostPort. */
containerPort: number;
/** Port on the host. */
hostPort: number;
/**
* Host address to listen on.
* @default "0.0.0.0"
*/
listenAddress?: string;
/** @default "TCP" */
protocol?: "TCP" | "UDP" | "SCTP";
}
/** A host path mounted into a kind node container. */
export interface KindMount {
/** Path on the host. */
hostPath: string;
/** Path in the node container. */
containerPath: string;
readOnly?: boolean;
/** Relabel the mount for SELinux hosts. */
selinuxRelabel?: boolean;
propagation?: "None" | "HostToContainer" | "Bidirectional";
}
/** Cluster networking settings for kind. */
export interface KindNetworking {
ipFamily?: "ipv4" | "ipv6" | "dual";
/**
* Host port of the API server.
* @default a random port
*/
apiServerPort?: number;
/**
* Host address the API server listens on.
* @default "127.0.0.1"
*/
apiServerAddress?: string;
/** Pod network CIDR. */
podSubnet?: string;
/** Service network CIDR. */
serviceSubnet?: string;
/** Skip kind's default CNI so you can install your own. */
disableDefaultCNI?: boolean;
kubeProxyMode?: "iptables" | "ipvs" | "nftables" | "none";
/** DNS search domains for nodes. */
dnsSearch?: string[];
}
/** An RFC 6902 JSON patch targeting a kubeadm config object. */
export interface KindJsonPatch {
group: string;
version: string;
kind: string;
/** The JSON patch document, as YAML or JSON. */
patch: string;
}
export interface LocalCluster extends Resource<
"Kubernetes.LocalCluster",
LocalClusterProps,
{
/** The kind cluster name. */
name: string;
/** The kubeconfig context of the cluster (`kind-<name>`). */
context: string;
/** The kubeconfig file holding the context, if not the default. */
kubeconfig: string | undefined;
/** The cluster's image registry (`localhost:<registryPort>`). */
registry: ContainerRegistry;
/** Name of the registry's Docker container. */
registryContainer: string;
/** CPU architecture of the cluster's nodes (the Docker host's). */
architecture: "amd64" | "arm64";
/**
* Connection for `Kubernetes.*` resources, including the registry and
* node architecture. Pass the whole resource as their `cluster`.
*/
connection: Connection;
},
{},
Providers
> {}
/**
* A Kubernetes cluster on your machine, for trying Alchemy's Kubernetes
* resources and developing against them without a hosted cluster.
*
* `LocalCluster` runs a [kind](https://kind.sigs.k8s.io) cluster in Docker
* together with a local image registry wired into the cluster's nodes, so
* workloads built from a `main` Effect program or a `context` Dockerfile
* work the same way they do on a hosted cluster with a registry. Pass the
* resource as the `cluster` of any `Kubernetes.*` resource.
*
* Requires Docker and the `kind` CLI on the deploying machine (set
* `KIND_BIN` to use a specific binary). The cluster's kubeconfig context,
* `kind-<name>`, works with `kubectl`.
*
* ### Creating a Local Cluster
* **Example:** Cluster and a Deployment
* ```typescript
* const cluster = yield* Kubernetes.LocalCluster("Cluster", {
* name: "alchemy",
* });
*
* const web = yield* Kubernetes.Deployment("Web", {
* cluster,
* image: "ghcr.io/stefanprodan/podinfo:6.15.0",
* port: 9898,
* serviceType: "ClusterIP",
* });
* ```
*
* ### Effect Programs
* **Example:** Run an Effect program as a Job
* ```typescript
* const cluster = yield* Kubernetes.LocalCluster("Cluster", {
* name: "alchemy",
* });
*
* const hello = yield* Kubernetes.Job(
* "Hello",
* { cluster, main: import.meta.url },
* Effect.gen(function* () {
* return {
* run: Effect.log("hello from a Job"),
* };
* }),
* );
* ```
*
* ### Kind Configuration
* **Example:** Forward a host port to a NodePort
* ```typescript
* const cluster = yield* Kubernetes.LocalCluster("Cluster", {
* name: "alchemy",
* nodes: [
* {
* role: "control-plane",
* extraPortMappings: [{ containerPort: 30080, hostPort: 8080 }],
* },
* ],
* });
* ```
*
* ### Registry Port
* **Example:** Two local clusters
* ```typescript
* const blue = yield* Kubernetes.LocalCluster("Blue", { name: "blue" });
* const green = yield* Kubernetes.LocalCluster("Green", {
* name: "green",
* registryPort: 5002,
* });
* ```
*
* @resource
* @product LocalCluster
*/
export const LocalCluster = Resource<LocalCluster>("Kubernetes.LocalCluster");
export class LocalClusterError extends Data.TaggedError("Kubernetes.LocalClusterError")<{
message: string;
}> {}
const KindBin = Config.String("KIND_BIN").pipe(Effect.orElseSucceed(() => "kind"));
const DockerBin = Config.String("DOCKER_BIN").pipe(Effect.orElseSucceed(() => "docker"));
const REGISTRY_IMAGE = "registry:3";
const DEFAULT_REGISTRY_PORT = 5001;
interface CommandResult {
exitCode: number;
stdout: string;
stderr: string;
}
const toArchitecture = (arch: string): "amd64" | "arm64" =>
/^(arm64|aarch64)$/i.test(arch.trim()) ? "arm64" : "amd64";
const sanitizeName = (name: string) =>
name
.toLowerCase()
.replaceAll(/[^a-z0-9-]/g, "-")
.replaceAll(/^-+|-+$/g, "");
interface RegistryInspect {
State?: { Running?: boolean };
Config?: { Labels?: Record<string, string> | null };
HostConfig?: {
PortBindings?: Record<string, Array<{ HostPort?: string }> | null> | null;
};
NetworkSettings?: { Networks?: Record<string, unknown> | null };
}
export const LocalClusterProvider = () =>
Provider.effect(
LocalCluster,
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
/** Run a CLI, returning its exit code and output. */
const run = (bin: string, args: string[], stdin?: string) =>
ChildProcess.make(bin, args, {
stdin: stdin === undefined ? "ignore" : Stream.succeed(new TextEncoder().encode(stdin)),
stdout: "pipe",
stderr: "pipe",
detached: false,
extendEnv: true,
}).pipe(
(command) => spawner.spawn(command),
Effect.flatMap((child) =>
Effect.all(
{
exitCode: child.exitCode,
stdout: child.stdout.pipe(Stream.decodeText, Stream.mkString),
stderr: child.stderr.pipe(Stream.decodeText, Stream.mkString),
},
{ concurrency: "unbounded" },
),
),
Effect.map((result): CommandResult => ({
exitCode: Number(result.exitCode),
stdout: result.stdout.trim(),
stderr: result.stderr.trim(),
})),
Effect.scoped,
// A spawn failure (almost always ENOENT) means the CLI itself is
// missing — a machine-setup problem, not a resource error.
Effect.catchCause((cause) =>
Effect.die(
new Error(
`Failed to run '${bin}': ${String(cause)}. ` +
"Kubernetes.LocalCluster needs Docker and the kind CLI " +
"(https://kind.sigs.k8s.io/docs/user/quick-start/#installation). " +
"Set KIND_BIN or DOCKER_BIN to use specific binaries.",
),
),
),
);
/** Run a CLI and fail with its stderr on a non-zero exit. */
const exec = Effect.fn(function* (bin: string, args: string[], stdin?: string) {
const result = yield* run(bin, args, stdin);
if (result.exitCode !== 0) {
return yield* new LocalClusterError({
message: `${bin} ${args.join(" ")} exited with code ${result.exitCode}: ${result.stderr}`,
});
}
return result;
});
const kind = Effect.fn(function* (args: string[]) {
return yield* exec(yield* KindBin, args);
});
const docker = Effect.fn(function* (args: string[], stdin?: string) {
return yield* exec(yield* DockerBin, args, stdin);
});
const kubeconfigArgs = (kubeconfig: string | undefined) =>
kubeconfig !== undefined ? ["--kubeconfig", kubeconfig] : [];
const clusterName = (
id: string,
props: LocalClusterProps | undefined,
output: LocalCluster["Attributes"] | undefined,
) =>
output?.name !== undefined
? Effect.succeed(output.name)
: props?.name !== undefined
? Effect.succeed(sanitizeName(props.name))
: createPhysicalName({ id, maxLength: 40, lowercase: true }).pipe(
Effect.map(sanitizeName),
);
const clusterExists = Effect.fn(function* (name: string) {
const { stdout } = yield* kind(["get", "clusters"]);
return stdout.split(/\r?\n/).some((line) => line.trim() === name);
});
const inspectRegistry = Effect.fn(function* (container: string) {
const result = yield* run(yield* DockerBin, ["container", "inspect", container]);
if (result.exitCode !== 0) return undefined;
const [info] = JSON.parse(result.stdout) as RegistryInspect[];
return info;
});
const registryHostPort = (info: RegistryInspect | undefined) => {
const binding = info?.HostConfig?.PortBindings?.["5000/tcp"]?.[0];
return binding?.HostPort !== undefined ? Number(binding.HostPort) : undefined;
};
const hostArchitecture = Effect.gen(function* () {
const { stdout } = yield* docker(["version", "--format", "{{.Server.Arch}}"]);
return toArchitecture(stdout);
});
const toAttributes = (options: {
name: string;
kubeconfig: string | undefined;
registryPort: number;
architecture: "amd64" | "arm64";
}): LocalCluster["Attributes"] => {
const context = `kind-${options.name}`;
const registry = { server: `localhost:${options.registryPort}` };
return {
name: options.name,
context,
kubeconfig: options.kubeconfig,
registry,
registryContainer: `${options.name}-registry`,
architecture: options.architecture,
connection: {
auth: {
kind: "kubeconfig",
path: options.kubeconfig,
context,
},
registry,
architecture: options.architecture,
},
};
};
/** Ensure the kind cluster exists and its context is in the kubeconfig. */
const ensureCluster = Effect.fn(function* (options: {
name: string;
nodeImage: string | undefined;
kubeconfig: string | undefined;
config: KindClusterConfig;
session: { note: (message: string) => Effect.Effect<void> };
}) {
if (!(yield* clusterExists(options.name))) {
yield* options.session.note(
`Creating kind cluster ${options.name} (about 30 seconds)...`,
);
const dir = yield* fs.makeTempDirectory({
prefix: "alchemy-kind-",
});
const configFile = path.join(dir, "kind.yaml");
yield* fs.writeFileString(
configFile,
JSON.stringify(kindClusterConfig(options.config), null, 2),
);
yield* kind([
"create",
"cluster",
"--name",
options.name,
"--config",
configFile,
"--wait",
"120s",
...(options.nodeImage !== undefined ? ["--image", options.nodeImage] : []),
...kubeconfigArgs(options.kubeconfig),
]).pipe(
// A concurrent create of the same cluster is a race, not a failure.
Effect.catchTag("Kubernetes.LocalClusterError", (error) =>
/already exist/i.test(error.message) ? Effect.void : Effect.fail(error),
),
);
yield* fs.remove(dir, { recursive: true });
}
// Restore the context if it was removed from the kubeconfig.
yield* kind([
"export",
"kubeconfig",
"--name",
options.name,
...kubeconfigArgs(options.kubeconfig),
]);
});
/**
* Ensure the registry container runs on the desired port, sits on the
* `kind` network, and every node's containerd resolves
* `localhost:<port>` to it.
*/
const ensureRegistry = Effect.fn(function* (options: {
id: string;
name: string;
registryPort: number;
session: { note: (message: string) => Effect.Effect<void> };
}) {
const container = `${options.name}-registry`;
let info = yield* inspectRegistry(container);
if (info !== undefined && registryHostPort(info) !== options.registryPort) {
yield* docker(["container", "rm", "--force", container]);
info = undefined;
}
if (info === undefined) {
yield* options.session.note(
`Starting image registry on localhost:${options.registryPort}...`,
);
const labels = yield* createInternalTags(options.id);
yield* docker([
"container",
"run",
"--detach",
"--restart=always",
"--publish",
`127.0.0.1:${options.registryPort}:5000`,
"--network",
"bridge",
"--name",
container,
...Object.entries(labels).flatMap(([key, value]) => ["--label", `${key}=${value}`]),
REGISTRY_IMAGE,
]);
info = yield* inspectRegistry(container);
} else if (info.State?.Running !== true) {
yield* docker(["container", "start", container]);
}
if (info?.NetworkSettings?.Networks?.kind === undefined) {
yield* docker(["network", "connect", "kind", container]).pipe(
Effect.catchTag("Kubernetes.LocalClusterError", (error) =>
/already exists/i.test(error.message) ? Effect.void : Effect.fail(error),
),
);
}
// containerd on each node: pull `localhost:<port>/…` from the
// registry container over the kind network.
const registryDir = `/etc/containerd/certs.d/localhost:${options.registryPort}`;
const { stdout } = yield* kind(["get", "nodes", "--name", options.name]);
const nodes = stdout
.split(/\r?\n/)
.map((line) => line.trim())
.filter((line) => line.length > 0);
yield* Effect.forEach(
nodes,
(node) =>
Effect.gen(function* () {
yield* docker(["exec", node, "mkdir", "-p", registryDir]);
yield* docker(
["exec", "-i", node, "cp", "/dev/stdin", `${registryDir}/hosts.toml`],
`[host."http://${container}:5000"]\n`,
);
}),
{ concurrency: "unbounded" },
);
});
return {
stables: ["name", "context", "kubeconfig", "registryContainer"],
list: () => Effect.succeed([] as LocalCluster["Attributes"][]),
diff: Effect.fn(function* ({ id, olds, news, output }) {
if (!isResolved(news)) return;
const oldName = yield* clusterName(id, olds, output);
const newName = yield* clusterName(id, news, undefined);
if (
(news.name !== undefined && oldName !== newName) ||
olds?.nodeImage !== news.nodeImage ||
olds?.kubeconfig !== news.kubeconfig ||
!deepEqual(kindConfigOf(olds ?? {}), kindConfigOf(news), {
stripNullish: true,
})
) {
return { action: "replace", deleteFirst: true } as const;
}
}),
read: Effect.fn(function* ({ id, olds, output, previous }) {
const name = yield* clusterName(id, olds, output);
if (!(yield* clusterExists(name))) return undefined;
const registryInfo = yield* inspectRegistry(`${name}-registry`);
const attrs = toAttributes({
name,
kubeconfig: output?.kubeconfig ?? olds?.kubeconfig,
registryPort:
registryHostPort(registryInfo) ?? olds?.registryPort ?? DEFAULT_REGISTRY_PORT,
architecture: yield* hostArchitecture,
});
if (output) return attrs;
// kind clusters carry no ownership marker; the registry container
// we start next to them does. Anything else is foreign.
const owned = yield* hasAlchemyTags(id, registryInfo?.Config?.Labels ?? undefined);
return owned ? attrs : unownedUnlessPrevious(attrs, previous, "name");
}),
reconcile: Effect.fn(function* ({ id, news, output, session }) {
const name = yield* clusterName(id, news, output);
const registryPort = news.registryPort ?? DEFAULT_REGISTRY_PORT;
yield* ensureCluster({
name,
nodeImage: news.nodeImage,
kubeconfig: news.kubeconfig,
config: kindConfigOf(news),
session,
});
yield* ensureRegistry({ id, name, registryPort, session });
const attrs = toAttributes({
name,
kubeconfig: news.kubeconfig,
registryPort,
architecture: yield* hostArchitecture,
});
// Advertise the registry to in-cluster tooling (KEP-1755).
const transport = yield* connectCluster(attrs.connection);
yield* applyObject({
transport,
object: {
apiVersion: "v1",
kind: "ConfigMap",
metadata: {
name: "local-registry-hosting",
namespace: "kube-public",
},
data: {
"localRegistryHosting.v1": `host: "${attrs.registry.server}"\nhelp: "https://kind.sigs.k8s.io/docs/user/local-registry/"\n`,
},
},
});
yield* session.note(
`kind cluster ${name} ready (context ${attrs.context}, registry ${attrs.registry.server})`,
);
return attrs;
}),
delete: Effect.fn(function* ({ output }) {
// Both commands are idempotent: kind ignores a missing cluster.
yield* kind([
"delete",
"cluster",
"--name",
output.name,
...kubeconfigArgs(output.kubeconfig),
]);
yield* run(yield* DockerBin, ["container", "rm", "--force", output.registryContainer]);
}),
};
}),
);