sailfork / t3code-github / .repos/ alchemy-effect/ packages/ hetzner/ src/ hosted.ts

import * as Hetzner from "@distilled.cloud/hetzner";
import * as Bundle from "alchemy/Bundle/Bundle";
import {
  matchesPackageRoot,
  normalizeInstallTargets,
  resolvePackageInstallIdentity,
  type PackageInstall,
} from "alchemy/Bundle/InstalledPackages";
import { findCwdForBundle, resolveMainPath } from "alchemy/Bundle/TempRoot";
import type { ResourceBinding } from "alchemy/Resource";
import { createHostRuntimeContext, type HostRuntimeContext } from "alchemy/Runtime/Process";
import {
  contextRootOf,
  extraFileDestination,
  hashExtraFiles,
  isContextRootDest,
  posixRelUnder,
  resolveExtraSource,
} from "alchemy/Util/extraFiles";
import { sha256, sha256Object } from "alchemy/Util/sha256";
import { zipCode, zipFiles, type ZipFile } from "alchemy/Util/zip";
import * as Data from "effect/Data";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Path from "effect/Path";
import * as Schedule from "effect/Schedule";
import type * as rolldown from "rolldown";
import { waitForAction } from "./actions.ts";
import type { ServiceBinding } from "./MountVolume.ts";
import { openSshClient, type SshClient } from "./Ssh.ts";

export type HetznerHostRuntimeContext = HostRuntimeContext;

class VolumeAttachTimeout extends Data.TaggedError("Hetzner.VolumeAttachTimeout")<{
  volumeId: number;
  serverId: number;
}> {}

export const createHetznerHostRuntimeContext = createHostRuntimeContext;

export interface HetznerBuildOptions extends Bundle.BundleConfig {
  /**
   * Native or Node-only packages to `npm install` into the unit instead
   * of bundling them. Same shape as Fly/Lambda `build.install`.
   */
  readonly install?: PackageInstall;
}

export interface HostedProgramProps {
  main: string;
  handler?: string;
  port?: number;
  env?: Record<string, any>;
  isExternal?: boolean;
  build?: HetznerBuildOptions;
  /**
   * Extra host directories packed into the unit archive (framework
   * client assets, Next.js `.next`, …). Hashed into `code.hash` so
   * asset changes update the unit. Destination is relative to the
   * unit root.
   */
  extraFiles?: ReadonlyArray<ExtraFile>;
}

export interface ExtraFile {
  readonly source: string;
  readonly destination: string;
}

const matchesConfiguredExternal = (
  external: rolldown.InputOptions["external"],
  moduleId: string,
  parentId: string | undefined,
  isResolved: boolean,
): boolean => {
  if (external === undefined) return false;
  if (typeof external === "function") {
    return external(moduleId, parentId, isResolved) === true;
  }
  const matchers = Array.isArray(external) ? external : [external];
  return matchers.some((matcher) =>
    typeof matcher === "string" ? matcher === moduleId : matcher.test(moduleId),
  );
};

export { extraFileDestination };

const quoteEnvValue = (value: unknown) => {
  const text = typeof value === "string" ? value : JSON.stringify(value ?? null);
  return `'${text.replaceAll(/'/g, `'""'`).replaceAll(/\n/g, "\\n")}'`;
};

export const renderEnvFile = (env: Record<string, unknown>) =>
  Object.entries(env)
    .sort(([a], [b]) => a.localeCompare(b))
    .map(([key, value]) => `${key}=${quoteEnvValue(value)}`)
    .join("\n");

export const collectBindingState = (bindings: ResourceBinding<ServiceBinding>[]) => {
  const active = bindings.filter(
    (binding: ResourceBinding<ServiceBinding> & { action?: string }) => binding.action !== "delete",
  );
  const env = active
    .map((binding) => binding?.data?.env)
    .reduce<Record<string, any>>((acc, value) => ({ ...acc, ...value }), {});
  const volumes: Array<{ volumeId: number; path: string }> = [];
  const seen = new Set<string>();
  for (const binding of active) {
    for (const volume of binding?.data?.volumes ?? []) {
      const key = `${volume.volumeId}:${volume.path}`;
      if (seen.has(key)) continue;
      seen.add(key);
      volumes.push(volume);
    }
  }
  return { env, volumes };
};

const skipExtraPath = (relative: string) =>
  relative
    .replaceAll("\\", "/")
    .split("/")
    .some(
      (segment) =>
        segment === "node_modules" || segment === ".git" || segment.startsWith(".alchemy-hetzner"),
    );

const toSharedExtraFiles = (extraFiles: ReadonlyArray<ExtraFile> | undefined) =>
  extraFiles?.map((file) => ({
    source: file.source,
    dest: file.destination,
  }));

const collectExtraFiles = Effect.fn(function* (extraFiles: ReadonlyArray<ExtraFile> | undefined) {
  if (extraFiles === undefined || extraFiles.length === 0) {
    return [] as ZipFile[];
  }
  const fs = yield* FileSystem.FileSystem;
  const path = yield* Path.Path;
  const files: ZipFile[] = [];
  for (const extra of extraFiles) {
    const dest = extraFileDestination(extra.destination);
    const source = resolveExtraSource(extra.source, path);
    if (!(yield* fs.exists(source))) continue;
    const rootStat = yield* fs.stat(source);
    const skip = isContextRootDest(dest)
      ? (relative: string) =>
          relative
            .replaceAll("\\", "/")
            .split("/")
            .some((segment) => segment === ".git" || segment.startsWith(".alchemy-hetzner"))
      : skipExtraPath;
    if (rootStat.type === "File") {
      files.push({
        path: isContextRootDest(dest) ? path.basename(source) : dest,
        content: yield* fs.readFile(source),
      });
      continue;
    }
    const names = yield* fs.readDirectory(source, { recursive: true });
    for (const name of names) {
      if (skip(name)) continue;
      const full = path.join(source, name);
      const stat = yield* fs.stat(full);
      if (stat.type !== "File") continue;
      const posixName = name.replaceAll("\\", "/");
      files.push({
        path: isContextRootDest(dest) ? posixName : `${dest}/${posixName}`.replaceAll(/\/+/g, "/"),
        content: yield* fs.readFile(full),
      });
    }
  }
  return files;
});

/**
 * The generated entry for `Hetzner.Service` units: a shim importing only
 * `@alchemy.run/hetzner/Bootstrap` plus the user's `main` — see that
 * module for why the entry never imports alchemy's own dependencies.
 */
const makeBunBootstrap =
  (handler: string) =>
  (importPath: string): string =>
    `
import { bootstrap } from "@alchemy.run/hetzner/Bootstrap";
import { ${handler} as entrypoint } from ${JSON.stringify(importPath)};

await bootstrap(entrypoint);
`;

export const createHetznerHostedSupport = ({
  stackName,
  stage,
  virtualEntryPlugin,
}: {
  stackName: string;
  stage: string;
  virtualEntryPlugin: (content: (importPath: string) => string) => rolldown.Plugin;
}) => {
  const alchemyEnv = {
    ALCHEMY_STACK_NAME: stackName,
    ALCHEMY_STAGE: stage,
    ALCHEMY_PHASE: "runtime",
    HOST: "0.0.0.0",
  };

  const bundleProgram = Effect.fn(function* (_id: string, props: HostedProgramProps) {
    const handler = props.handler ?? "default";
    const realMain = yield* resolveMainPath(props.main);
    const cwd = yield* findCwdForBundle(realMain);

    const requested = yield* normalizeInstallTargets(props.build?.install);
    const installRoots = new Set(Object.keys(requested));
    const configuredExternal = props.build?.input?.external;

    const buildBundle = Effect.fn(function* (
      entry: string,
      plugins?: rolldown.RolldownPluginOption,
    ) {
      return yield* Bundle.build(
        {
          ...props.build?.input,
          input: entry,
          cwd,
          platform: "node",
          external: (moduleId, parentId, isResolved) => {
            if (moduleId === "bun" || moduleId.startsWith("bun:")) return true;
            for (const root of installRoots) {
              if (matchesPackageRoot(moduleId, root)) return true;
            }
            return matchesConfiguredExternal(configuredExternal, moduleId, parentId, isResolved);
          },
          resolve: {
            conditionNames: [...Bundle.NODE_CONDITION_NAMES],
            ...props.build?.input?.resolve,
          },
          plugins: [props.build?.input?.plugins, plugins],
        },
        {
          ...props.build?.output,
          format: "esm",
          sourcemap: props.build?.output?.sourcemap ?? false,
          minify: props.build?.output?.minify ?? false,
          entryFileNames: "index.mjs",
          // Preserve init order for framework SSR bundles.
          strictExecutionOrder: true,
          keepNames: true,
        },
        props.build,
      );
    });

    const toBytes = (content: string | Uint8Array<ArrayBufferLike>) =>
      typeof content === "string" ? new TextEncoder().encode(content) : content;
    const extraFiles = yield* collectExtraFiles(props.extraFiles);
    const identity =
      Object.keys(requested).length > 0
        ? yield* resolvePackageInstallIdentity({ cwd, requested })
        : undefined;
    const install =
      identity !== undefined && Object.keys(identity.resolved).length > 0
        ? identity.resolved
        : undefined;
    const packageJson =
      install === undefined
        ? undefined
        : `${JSON.stringify({ private: true, type: "module", dependencies: install }, null, 2)}\n`;
    const installFiles =
      packageJson !== undefined
        ? [
            {
              path: "package.json",
              content: new TextEncoder().encode(packageJson),
            },
          ]
        : [];

    // `isExternal` mains are already complete Node programs. Pack the
    // dist tree as-built so relative imports (`../client`, nitro chunks)
    // survive. Do not rolldown-flatten.
    if (props.isExternal) {
      const fs = yield* FileSystem.FileSystem;
      const pathMod = yield* Path.Path;
      const extras = toSharedExtraFiles(props.extraFiles) ?? [];
      const root = contextRootOf(realMain, extras, pathMod, (source) =>
        resolveExtraSource(source, pathMod),
      );
      const entryRel = posixRelUnder(root, realMain, pathMod) ?? pathMod.basename(realMain);
      const entryBytes = yield* fs.readFile(realMain);
      const zipEntries: ZipFile[] = [
        ...extraFiles.filter(
          (file) => !(file.path === "package.json" && packageJson !== undefined),
        ),
        ...installFiles,
      ];
      if (!zipEntries.some((file) => file.path === entryRel)) {
        zipEntries.push({ path: entryRel, content: entryBytes });
      }
      const archive = yield* zipFiles(zipEntries);
      const extraHash = yield* hashExtraFiles(toSharedExtraFiles(props.extraFiles));
      const hash = yield* sha256Object({
        bundle: yield* sha256(entryBytes),
        extra: extraHash,
        packageJson: packageJson ?? "",
        entryRel,
      });
      return { archive, hash, entryRel };
    }

    const bundleOutput = yield* buildBundle(
      realMain,
      virtualEntryPlugin(makeBunBootstrap(handler)),
    );
    const [entryFile, ...chunkFiles] = bundleOutput.files;
    const zipExtras = [
      ...chunkFiles.map((file) => ({
        path: file.path,
        content: toBytes(file.content),
      })),
      ...extraFiles,
      ...installFiles,
    ];
    const archive = yield* zipCode(toBytes(entryFile.content), zipExtras);
    const extraHash = yield* hashExtraFiles(toSharedExtraFiles(props.extraFiles));
    const hash = yield* sha256Object({
      bundle: bundleOutput.hash,
      extra: extraHash,
      packageJson: packageJson ?? "",
    });
    return { archive, hash, entryRel: "index.mjs" };
  });

  const renderUnit = (unitName: string, appDir: string, entryRel: string) => `[Unit]
Description=Alchemy Hetzner Service ${unitName}
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
WorkingDirectory=${appDir}
EnvironmentFile=-${appDir}/env
Environment=HOST=0.0.0.0
ExecStart=/usr/bin/env node ${appDir}/${entryRel}
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
`;

  const waitForSsh = (ssh: SshClient) =>
    ssh.exec("true").pipe(
      Effect.retry({
        while: (e) => e._tag === "Hetzner.SshError",
        times: 10,
        schedule: Schedule.min([
          Schedule.exponential(Duration.millis(500), 1.5),
          Schedule.spaced(Duration.seconds(5)),
        ]),
      }),
    );

  const hetznerTransient = (e: { readonly _tag: string }): boolean =>
    e._tag === "TooManyRequests" ||
    e._tag === "ServiceUnavailable" ||
    e._tag === "InternalServerError" ||
    e._tag === "BadGateway" ||
    e._tag === "GatewayTimeout" ||
    e._tag === "Locked";

  const attachAndMount = Effect.fn(function* (input: {
    ssh: SshClient;
    serverId: number;
    volumes: Array<{ volumeId: number; path: string }>;
  }) {
    for (const { volumeId, path } of input.volumes) {
      let volume = yield* Hetzner.volumes.getVolume({ id: volumeId }).pipe(
        Effect.map(({ volume }) => volume),
        Effect.retry({
          while: hetznerTransient,
          times: 8,
          schedule: Schedule.min([
            Schedule.exponential(Duration.millis(500), 1.5),
            Schedule.spaced(Duration.seconds(5)),
          ]),
        }),
        Effect.catchTag("NotFound", () => Effect.undefined),
      );
      if (volume === undefined) continue;

      if (volume.server !== input.serverId) {
        if (volume.server !== null) {
          yield* Hetzner.volumeActions.detachVolume({ id: volumeId }).pipe(
            Effect.tap(({ action }) =>
              waitForAction(action).pipe(Effect.catchTag("ActionTimeout", () => Effect.void)),
            ),
            Effect.catchTag(
              ["NotFound", "UnprocessableEntity", "Locked", "Conflict"],
              () => Effect.void,
            ),
          );
        }
        yield* Hetzner.volumeActions
          .attachVolume({
            id: volumeId,
            server: input.serverId,
            // No automount: Hetzner would mount the device at
            // `/mnt/HC_Volume_{id}`, and a device mounted elsewhere makes
            // our own `mount` at the service path fail with
            // "already mounted". We own the mount + fstab below.
            automount: false,
          })
          .pipe(
            Effect.retry({
              while: hetznerTransient,
              times: 8,
              schedule: Schedule.min([
                Schedule.exponential(Duration.millis(500), 1.5),
                Schedule.spaced(Duration.seconds(5)),
              ]),
            }),
            Effect.tap(({ action }) =>
              waitForAction(action).pipe(Effect.catchTag("ActionTimeout", () => Effect.void)),
            ),
            Effect.catchTag(["UnprocessableEntity", "Locked", "Conflict"], () => Effect.void),
          );
        volume = yield* Hetzner.volumes.getVolume({ id: volumeId }).pipe(
          Effect.flatMap(({ volume }) =>
            volume.server === input.serverId
              ? Effect.succeed(volume)
              : Effect.fail({ _tag: "AttachPending" as const }),
          ),
          Effect.retry({
            while: (e) =>
              e._tag === "AttachPending" || e._tag === "TooManyRequests" || e._tag === "Locked",
            times: 10,
            schedule: Schedule.min([
              Schedule.exponential(Duration.millis(500), 1.5),
              Schedule.spaced(Duration.seconds(5)),
            ]),
          }),
          Effect.catchTags({
            AttachPending: () =>
              Hetzner.volumes.getVolume({ id: volumeId }).pipe(
                Effect.map(({ volume }) => volume),
                Effect.catchTag("NotFound", () => Effect.undefined),
              ),
            NotFound: () => Effect.undefined,
          }),
        );
      }
      if (volume === undefined || volume.server !== input.serverId) {
        return yield* new VolumeAttachTimeout({
          volumeId,
          serverId: input.serverId,
        });
      }

      const device = volume.linux_device;
      const fsType = volume.format === "xfs" ? "xfs" : "ext4";
      const pathLit = JSON.stringify(path);
      const deviceLit = JSON.stringify(device);
      const fsLit = JSON.stringify(fsType);
      // Attach returns before udev creates `/dev/disk/by-id/scsi-0HC_Volume_*`.
      yield* input.ssh.exec(
        [
          `set -euo pipefail`,
          `mkdir -p ${pathLit}`,
          `if ! findmnt -n ${pathLit} >/dev/null 2>&1; then`,
          `  udevadm settle --timeout=10 2>/dev/null || true`,
          `  mounted=0`,
          `  for i in 1 2 3 4 5 6 7 8 9 10; do`,
          // A sibling Service sharing the same (volume, path) may have
          // mounted it between iterations — that IS the desired state.
          `    if findmnt -n ${pathLit} >/dev/null 2>&1; then`,
          `      mounted=1`,
          `      break`,
          `    fi`,
          `    if [ -e ${deviceLit} ]; then`,
          // A previous attach with automount (or a leaked mount from an
          // interrupted run) may hold the device at /mnt/HC_Volume_* —
          // "already mounted" would fail our mount at the service path.
          `      current="$(findmnt -n -o TARGET ${deviceLit} 2>/dev/null || true)"`,
          `      if [ -n "$current" ] && [ "$current" != ${pathLit} ]; then`,
          `        umount ${deviceLit} 2>/dev/null || true`,
          `      fi`,
          `      if mount -t ${fsLit} ${deviceLit} ${pathLit}; then`,
          `        mounted=1`,
          `        break`,
          `      fi`,
          `    fi`,
          `    sleep 2`,
          `  done`,
          `  if [ "$mounted" != 1 ]; then`,
          `    echo "volume ${deviceLit} could not be mounted at ${pathLit}" >&2`,
          `    ls -l /dev/disk/by-id 2>/dev/null || true`,
          `    exit 1`,
          `  fi`,
          `fi`,
          `if ! grep -qF ${pathLit} /etc/fstab; then`,
          `  echo ${JSON.stringify(`${device} ${path} ${fsType} defaults,nofail 0 2`)} >> /etc/fstab`,
          `fi`,
        ].join("\n"),
      );
    }
  });

  const deployUnit = Effect.fn(function* (input: {
    ssh: SshClient;
    unitName: string;
    archive: Uint8Array<ArrayBufferLike>;
    env: Record<string, unknown>;
    entryRel?: string;
  }) {
    const appDir = `/opt/${input.unitName}`;
    yield* waitForSsh(input.ssh);
    yield* input.ssh.exec(
      [
        `set -uo pipefail`,
        `export HOME=/root`,
        `mkdir -p ${JSON.stringify(appDir)}`,
        // A fresh server's cloud-init bootstrap (`ALCHEMY_BOOTSTRAP` in
        // `Server.ts`) installs these same packages and holds the dpkg lock
        // while it runs; let it finish instead of racing it.
        `if command -v cloud-init >/dev/null 2>&1; then cloud-init status --wait >/dev/null 2>&1 || true; fi`,
        `if ! command -v curl >/dev/null 2>&1 || ! command -v unzip >/dev/null 2>&1 || [ ! -f /etc/ssl/certs/ca-certificates.crt ]; then`,
        `  apt-get -o DPkg::Lock::Timeout=300 update`,
        `  DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300 install -y curl unzip ca-certificates`,
        `fi`,
        `for tool in curl unzip; do`,
        `  if ! command -v "$tool" >/dev/null 2>&1; then`,
        `    echo "$tool install failed" >&2`,
        `    exit 1`,
        `  fi`,
        `done`,
        `if ! command -v node >/dev/null 2>&1 || ! node -e 'process.exit(Number(process.versions.node.split(".")[0])<26)'; then`,
        `  for attempt in 1 2 3 4 5; do`,
        `    curl -fsSL https://deb.nodesource.com/setup_26.x | bash - && DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=300 install -y nodejs && break`,
        `    sleep 5`,
        `  done`,
        `fi`,
        `if ! command -v node >/dev/null 2>&1; then`,
        `  echo "node install failed" >&2`,
        `  exit 1`,
        `fi`,
      ].join("\n"),
    );
    yield* input.ssh.scp(input.archive, `${appDir}/bundle.zip`);
    yield* input.ssh.scp(new TextEncoder().encode(renderEnvFile(input.env)), `${appDir}/env`);
    yield* input.ssh.scp(
      new TextEncoder().encode(renderUnit(input.unitName, appDir, input.entryRel ?? "index.mjs")),
      `/etc/systemd/system/${input.unitName}.service`,
    );
    yield* input.ssh.exec(
      [
        `set -euo pipefail`,
        `export HOME=/root`,
        `if ! command -v node >/dev/null 2>&1; then echo "node missing at start" >&2; exit 1; fi`,
        `unzip -o ${JSON.stringify(`${appDir}/bundle.zip`)} -d ${JSON.stringify(appDir)}`,
        `if [ -f ${JSON.stringify(`${appDir}/package.json`)} ]; then`,
        `  (cd ${JSON.stringify(appDir)} && npm install --omit=dev --no-fund --no-audit)`,
        `fi`,
        `systemctl daemon-reload`,
        `systemctl enable --now ${input.unitName}.service`,
        `systemctl restart ${input.unitName}.service`,
      ].join("\n"),
    );
    const port = typeof input.env.PORT === "string" ? input.env.PORT : undefined;
    const health =
      port !== undefined ? `curl -sf -o /dev/null http://127.0.0.1:${port}/health` : "true";
    yield* input.ssh.exec(
      [
        `set -uo pipefail`,
        `for attempt in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30; do`,
        `  if systemctl is-active --quiet ${input.unitName}.service && ${health}; then`,
        `    exit 0`,
        `  fi`,
        `  sleep 2`,
        `done`,
        `echo "unit ${input.unitName} not ready" >&2`,
        `systemctl status ${input.unitName}.service --no-pager || true`,
        `journalctl -u ${input.unitName}.service -n 80 --no-pager || true`,
        `ls -la ${JSON.stringify(appDir)} >&2 || true`,
        `exit 1`,
      ].join("\n"),
    );
  });

  const removeUnit = Effect.fn(function* (input: { ssh: SshClient; unitName: string }) {
    const appDir = `/opt/${input.unitName}`;
    yield* input.ssh
      .exec(
        [
          `systemctl disable --now ${input.unitName}.service || true`,
          `rm -f /etc/systemd/system/${input.unitName}.service`,
          `rm -rf ${JSON.stringify(appDir)}`,
          `systemctl daemon-reload || true`,
        ].join("\n"),
      )
      .pipe(Effect.catchTag("Hetzner.SshError", () => Effect.void));
  });

  return {
    alchemyEnv,
    bundleProgram,
    attachAndMount,
    deployUnit,
    removeUnit,
    waitForSsh,
    openSshClient,
  };
};