sailfork / t3code-github / .repos/ alchemy-effect/ packages/ alchemy/ src/ State/ BucketState.ts

import * as Clock from "effect/Clock";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as Scope from "effect/Scope";
import * as Semaphore from "effect/Semaphore";
import { decodeFqn, encodeFqn } from "../FQN.ts";
import { STATE_STORE_VERSION } from "./HttpStateApi.ts";
import { StateStoreError, type PersistedState, type StateService } from "./State.ts";
import { encodeState, reviveState } from "./StateEncoding.ts";

/**
 * A state store over an object storage bucket (S3, Cloud Storage, …).
 *
 * The store is laid out exactly like the local state store's file tree,
 * with the bucket (plus an optional prefix) in place of `.alchemy/state`:
 *
 * ```
 * {prefix}{stack}/{stage}/{fqn}.json
 * {prefix}{stack}/{stage}/__stack_output__.json
 * {prefix}{stack}/{stage}/__lease__.json
 * ```
 *
 * Concurrent deploys of one stack and stage share the `__lease__.json`
 * object, written with conditional puts. The second deploy fails
 * immediately instead of mixing rows. The holder refreshes the lease until
 * the store closes; a killed process stops refreshing, the lease expires,
 * and the next deploy can take it.
 *
 * Each backend supplies a {@link BucketObjects}: plain object operations
 * plus a version token (an S3 ETag, a Cloud Storage generation) for the
 * conditional writes the lease needs.
 */

/** The bookkeeping object that stores a stack's resolved output. */
const OUTPUT_FILE = "__stack_output__.json";

/**
 * Per `(stack, stage)` lease. A delete of the stage leaves it in place so
 * the lock outlives the objects it was guarding; the store deletes it on
 * close.
 */
/** The lease token `alchemy state migrate` leaves on a stage it moved to DynamoDB. */
const MIGRATED_TOKEN = "migrated:dynamodb";

const LEASE_FILE = "__lease__.json";

/** How long a lease stays valid without a refresh. */
const LEASE_TTL_MS = 60_000;

/** How long a passing lease check is trusted. A failure is never cached. */
const LEASE_CHECK_TTL_MS = 5_000;

/** A write condition, in terms of the backend's version token. */
export type WriteCondition =
  /** Only create the object; fail if it exists. */
  | { readonly ifAbsent: true }
  /** Only overwrite the object at exactly this version. */
  | { readonly ifVersion: string };

/**
 * The object operations a bucket backend implements. Every operation is
 * self-contained (`R = never`) and fails with {@link StateStoreError}.
 */
export interface BucketObjects {
  /** Every key under `prefix`, across pagination. */
  readonly listKeys: (prefix: string) => Effect.Effect<ReadonlyArray<string>, StateStoreError>;
  /**
   * The immediate "directory" names under `prefix` (delimiter `/`), without
   * the prefix or the trailing slash.
   */
  readonly listChildren: (prefix: string) => Effect.Effect<ReadonlyArray<string>, StateStoreError>;
  /** An object's text and version, or `undefined` when it doesn't exist. */
  readonly read: (
    key: string,
  ) => Effect.Effect<
    { readonly text: string; readonly version: string } | undefined,
    StateStoreError
  >;
  /**
   * Write an object. Returns its new version, or `undefined` when
   * `condition` did not hold.
   */
  readonly write: (
    key: string,
    body: string,
    condition?: WriteCondition,
  ) => Effect.Effect<string | undefined, StateStoreError>;
  /**
   * Delete an object, only at `ifVersion` when given. Deleting a missing
   * object, or one at another version, succeeds without deleting.
   */
  readonly delete: (key: string, ifVersion?: string) => Effect.Effect<void, StateStoreError>;
  /** Delete many objects. Missing objects are ignored. */
  readonly deleteMany: (keys: ReadonlyArray<string>) => Effect.Effect<void, StateStoreError>;
}

export interface BucketStateOptions {
  /** The store's id, e.g. `"s3"` or `"gcs"`. */
  readonly id: string;
  /** How errors name the store, e.g. `"S3"` or `"Cloud Storage"`. */
  readonly label: string;
  /** Key prefix within the bucket. A trailing `/` is added. */
  readonly prefix?: string;
  /**
   * The bucket's objects. Runs once, on the first state operation, and is
   * cached: ensure the bucket exists here, never at construction.
   */
  readonly objects: Effect.Effect<BucketObjects, StateStoreError>;
}

/**
 * Tokens of the leases this process holds, so a conflict with one of them
 * can say the holder is an earlier, still-open build in the same process.
 */
const heldInProcess = new Set<string>();

interface LeaseRecord {
  readonly token: string;
  readonly expiresAt: number;
}

interface HeldLease {
  readonly checkLive: Effect.Effect<void, StateStoreError>;
}

/**
 * A passing check is trusted for `ttlMs`, so a burst of operations does
 * one round-trip. A failing check is never cached.
 */
const amortizeCheck = (
  checkLive: Effect.Effect<void, StateStoreError>,
  ttlMs: number,
): Effect.Effect<void, StateStoreError> => {
  let lastOkAt: number | undefined;
  return Effect.suspend(() => {
    if (lastOkAt !== undefined && Date.now() - lastOkAt < ttlMs) {
      return Effect.void;
    }
    return checkLive.pipe(
      Effect.tap(() =>
        Effect.sync(() => {
          lastOkAt = Date.now();
        }),
      ),
    );
  });
};

const parseLease = (text: string): LeaseRecord => {
  let parsed: unknown;
  try {
    parsed = JSON.parse(text);
  } catch {
    parsed = undefined;
  }
  return typeof parsed === "object" &&
    parsed !== null &&
    "token" in parsed &&
    "expiresAt" in parsed &&
    typeof parsed.token === "string" &&
    typeof parsed.expiresAt === "number"
    ? { token: parsed.token, expiresAt: parsed.expiresAt }
    : { token: "", expiresAt: 0 };
};

/**
 * Construct a bucket-backed {@link StateService}. Leases are bound to the
 * ambient scope; callers that need them to outlive the first operation
 * bind this effect to a longer scope with `Scope.provide`.
 */
export const makeBucketState = (options: BucketStateOptions) =>
  Effect.gen(function* () {
    const leaseScope = yield* Effect.scope;
    const prefix = options.prefix ? `${options.prefix.replace(/\/+$/, "")}/` : "";
    const objects = yield* Effect.cached(options.objects);
    const use = <A>(
      f: (objects: BucketObjects) => Effect.Effect<A, StateStoreError>,
    ): Effect.Effect<A, StateStoreError> => objects.pipe(Effect.flatMap(f));

    const stagePrefix = ({ stack, stage }: { stack: string; stage: string }) =>
      `${prefix}${stack}/${stage}/`;
    const resourceKey = (request: { stack: string; stage: string; fqn: string }) =>
      `${stagePrefix(request)}${encodeFqn(request.fqn)}.json`;
    const outputKey = (request: { stack: string; stage: string }) =>
      `${stagePrefix(request)}${OUTPUT_FILE}`;
    const leaseKey = (request: { stack: string; stage: string }) =>
      `${stagePrefix(request)}${LEASE_FILE}`;

    const readJson = <T>(key: string) =>
      use((o) => o.read(key)).pipe(
        Effect.flatMap((object) =>
          object === undefined
            ? Effect.undefined
            : Effect.try({
                try: () => JSON.parse(object.text, reviveState) as T,
                catch: (cause) =>
                  new StateStoreError({
                    message: `Failed to parse state object '${key}'`,
                    cause: cause instanceof Error ? cause : undefined,
                  }),
              }),
        ),
      );

    const writeJson = (key: string, value: unknown) =>
      use((o) => o.write(key, JSON.stringify(encodeState(value), null, 2)));

    /** Delete every object under `keyPrefix`, except leases. Idempotent. */
    const deleteAll = (keyPrefix: string) =>
      use((o) =>
        o.listKeys(keyPrefix).pipe(
          // Leave the lease object. Releasing it here would open the stage
          // before this delete finishes, and the store still holds the lock.
          Effect.map((keys) => keys.filter((key) => !key.endsWith(`/${LEASE_FILE}`))),
          Effect.flatMap((keys) => (keys.length === 0 ? Effect.void : o.deleteMany(keys))),
        ),
      );

    const heldError = (stack: string, stage: string, holder: string | undefined) =>
      // The message always starts "another deploy holds": callers match on it
      // (the AWS migration tells a held stage from a failed one by it).
      new StateStoreError({
        message:
          holder !== undefined && heldInProcess.has(holder)
            ? `another deploy holds the ${options.label} state lock '${stack}/${stage}': this process holds it, from an earlier operation on this stage that is still open. Run each operation in its own Effect.scoped so its build closes first.`
            : `another deploy holds the ${options.label} state lock '${stack}/${stage}'`,
      });

    const lostError = (stack: string, stage: string) =>
      new StateStoreError({
        message: `the ${options.label} state lock '${stack}/${stage}' was lost mid-run; refusing to continue unlocked`,
      });

    const readLease = (key: string) =>
      use((o) => o.read(key)).pipe(
        Effect.map((object) =>
          object === undefined
            ? undefined
            : { version: object.version, record: parseLease(object.text) },
        ),
      );

    const putLease = (key: string, record: LeaseRecord, condition: WriteCondition) =>
      use((o) => o.write(key, JSON.stringify(record), condition));

    const acquireLease = (stack: string, stage: string) =>
      Effect.gen(function* () {
        const key = leaseKey({ stack, stage });
        const token = yield* Effect.sync(() => crypto.randomUUID());
        const now = yield* Clock.currentTimeMillis;
        const fresh: LeaseRecord = { token, expiresAt: now + LEASE_TTL_MS };
        let version = yield* putLease(key, fresh, { ifAbsent: true });
        if (version === undefined) {
          const existing = yield* readLease(key);
          const stillHeld = existing === undefined || existing.record.expiresAt > now;
          if (existing?.record.token === MIGRATED_TOKEN) {
            return yield* new StateStoreError({
              message: `${heldError(stack, stage, undefined).message}: the stage moved to a DynamoDB state store. Deploy it with a version of alchemy that has AWS.state() on DynamoDB, or run 'alchemy state migrate --rollback' with that version to move it back.`,
            });
          }
          if (stillHeld) {
            return yield* heldError(stack, stage, existing?.record.token);
          }
          const replaced = yield* putLease(key, fresh, { ifVersion: existing.version });
          if (replaced === undefined) {
            return yield* heldError(stack, stage, undefined);
          }
          version = replaced;
        }

        heldInProcess.add(token);
        const cursor = { version, lost: false };
        const gate = Semaphore.makeUnsafe(1);

        const renew = Semaphore.withPermits(
          gate,
          1,
        )(
          Effect.gen(function* () {
            if (cursor.lost) return;
            const renewedAt = yield* Clock.currentTimeMillis;
            const next = yield* putLease(
              key,
              { token, expiresAt: renewedAt + LEASE_TTL_MS },
              { ifVersion: cursor.version },
            );
            if (next === undefined) cursor.lost = true;
            else cursor.version = next;
          }),
        ).pipe(
          Effect.retry({
            while: () => !cursor.lost,
            schedule: Schedule.spaced("1 second"),
            times: 3,
          }),
          Effect.catch(() =>
            Effect.sync(() => {
              cursor.lost = true;
            }),
          ),
        );

        const verify = Semaphore.withPermits(
          gate,
          1,
        )(
          Effect.gen(function* () {
            if (cursor.lost) return yield* lostError(stack, stage);
            const existing = yield* readLease(key);
            if (existing === undefined || existing.record.token !== token) {
              cursor.lost = true;
              return yield* lostError(stack, stage);
            }
            const observedAt = yield* Clock.currentTimeMillis;
            if (existing.record.expiresAt <= observedAt) {
              // The lease lapsed without anyone taking it: a refresh queued
              // behind a large deploy's writes missed its window. Take it
              // back at the version we read, so a deploy that took it in
              // the meantime wins and this one stops.
              const reclaimed = yield* putLease(
                key,
                { token, expiresAt: observedAt + LEASE_TTL_MS },
                { ifVersion: existing.version },
              );
              if (reclaimed === undefined) {
                cursor.lost = true;
                return yield* lostError(stack, stage);
              }
              cursor.version = reclaimed;
              return;
            }
            cursor.version = existing.version;
          }),
        );

        // Interrupt the refresh before releasing, so a refresh cannot
        // land after the lease has been released.
        yield* Scope.addFinalizer(
          leaseScope,
          Semaphore.withPermits(
            gate,
            1,
          )(
            Effect.gen(function* () {
              cursor.lost = true;
              heldInProcess.delete(token);
              // Read the version we actually hold. A refresh that landed
              // after the cursor was last stored would make a release
              // against the old version a no-op, and the lease would stay.
              const existing = yield* readLease(key);
              if (existing === undefined || existing.record.token !== token) return;
              // Delete rather than expire: a leftover lease object would keep
              // a destroyed stage listed by `listStages`.
              yield* use((o) => o.delete(key, existing.version));
            }).pipe(Effect.ignore),
          ),
        );
        yield* renew.pipe(
          Effect.delay(Duration.millis(Math.floor(LEASE_TTL_MS / 3))),
          Effect.forever,
          Effect.interruptible,
          Effect.forkIn(leaseScope),
        );

        return { checkLive: amortizeCheck(verify, LEASE_CHECK_TTL_MS) } satisfies HeldLease;
      });

    // One lease per (stack, stage), taken on the first operation and held
    // for the store's lifetime. The mutex keeps two concurrent first
    // operations in this process from contending with each other.
    const leaseMutex = Semaphore.makeUnsafe(1);
    const leases = new Map<string, Effect.Effect<HeldLease, StateStoreError>>();

    const leaseFor = (stack: string, stage: string): Effect.Effect<HeldLease, StateStoreError> =>
      Semaphore.withPermits(
        leaseMutex,
        1,
      )(
        Effect.gen(function* () {
          const key = `${stack}/${stage}`;
          const existing = leases.get(key);
          if (existing !== undefined) return existing;
          // Drop a failed acquire. Caching it would make the next operation
          // report the same holder after that holder has gone.
          const cached = yield* Effect.cached(
            acquireLease(stack, stage).pipe(
              Effect.tapError(() =>
                Effect.sync(() => {
                  if (leases.get(key) === cached) {
                    leases.delete(key);
                  }
                }),
              ),
            ),
          );
          leases.set(key, cached);
          return cached;
        }),
      ).pipe(Effect.flatMap((lease) => lease));

    const guarded = <A>(
      request: { stack: string; stage: string },
      op: Effect.Effect<A, StateStoreError>,
    ): Effect.Effect<A, StateStoreError> =>
      leaseFor(request.stack, request.stage).pipe(
        Effect.flatMap((lease) => lease.checkLive),
        Effect.andThen(op),
      );

    const verifyHeldLeases: Effect.Effect<void, StateStoreError> = Effect.suspend(() =>
      Effect.forEach(
        Array.from(leases.values()),
        (lease) => lease.pipe(Effect.flatMap((held) => held.checkLive)),
        { discard: true },
      ),
    );

    const listChildren = (keyPrefix: string) =>
      use((o) => o.listChildren(keyPrefix)).pipe(Effect.map((names) => Array.from(names)));

    const state: StateService = {
      id: options.id,
      getVersion: () => Effect.succeed(STATE_STORE_VERSION),
      listStacks: () => verifyHeldLeases.pipe(Effect.andThen(listChildren(prefix))),
      listStages: (stack: string) =>
        verifyHeldLeases.pipe(Effect.andThen(listChildren(`${prefix}${stack}/`))),
      get: (request) => guarded(request, readJson<PersistedState>(resourceKey(request))),
      getReplacedResources: Effect.fn(function* (request) {
        return (yield* Effect.forEach(yield* state.list(request), (fqn) =>
          state.get({
            stack: request.stack,
            stage: request.stage,
            fqn,
          }),
        )).filter((r) => r?.status === "replaced");
      }),
      set: (request) =>
        guarded(
          request,
          writeJson(resourceKey(request), request.value).pipe(Effect.as(request.value)),
        ),
      delete: (request) =>
        guarded(
          request,
          use((o) => o.delete(resourceKey(request))),
        ),
      // A stack-wide delete locks every stage it is about to remove, so a
      // concurrent deploy fails the lease instead of racing the delete.
      deleteStack: ({ stack, stage }) =>
        stage === undefined
          ? verifyHeldLeases.pipe(
              Effect.andThen(state.listStages(stack)),
              Effect.flatMap((stages) =>
                Effect.forEach(
                  stages,
                  (found) =>
                    guarded(
                      { stack, stage: found },
                      deleteAll(stagePrefix({ stack, stage: found })),
                    ),
                  { discard: true },
                ),
              ),
              Effect.andThen(deleteAll(`${prefix}${stack}/`)),
            )
          : guarded({ stack, stage }, deleteAll(stagePrefix({ stack, stage }))),
      list: (request) =>
        guarded(
          request,
          use((o) => o.listKeys(stagePrefix(request))).pipe(
            Effect.map((keys) =>
              keys
                .map((key) => key.slice(stagePrefix(request).length))
                // Filter bookkeeping before decoding — `decodeFqn` replaces
                // `__` with `/`, which would turn `__stack_output__` into
                // `/stack_output/` and slip past the filter.
                .filter(
                  (file) => file !== OUTPUT_FILE && file !== LEASE_FILE && file.endsWith(".json"),
                )
                .map((file) => decodeFqn(file.replace(/\.json$/, ""))),
            ),
          ),
        ),
      getOutput: (request) => guarded(request, readJson(outputKey(request))),
      setOutput: (request) =>
        guarded(
          request,
          writeJson(outputKey(request), request.value).pipe(Effect.as(request.value)),
        ),
    };
    return state;
  });