sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Queues/ Consumer.ts

import * as queues from "@distilled.cloud/cloudflare/queues";
import * as workers from "@distilled.cloud/cloudflare/workers";
import { isResolved } from "alchemy/Diff";
import * as ProviderLayer from "alchemy/Local/ProviderLayer";
import * as RpcProvider from "alchemy/Local/RpcProvider";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { Stack } from "alchemy/Stack";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as MutableHashMap from "effect/MutableHashMap";
import * as Option from "effect/Option";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
import { localAccountId } from "../LocalAccount.ts";
import {
  isLiveId,
  LOCAL_PROVIDERS_URL,
  LocalRuntimeState,
  localRuntimeServices,
} from "../LocalRuntime.ts";
import type { Providers } from "../Providers.ts";

export type ConsumerProps = {
  /**
   * The queue ID to attach the consumer to.
   */
  queueId: string;
  /**
   * Name of the Worker script that will consume messages.
   */
  scriptName: string;
  /**
   * Name of the queue that receives messages after `maxRetries` failed
   * deliveries.
   *
   * Cloudflare does not drain or alert on a dead-letter queue. A message
   * left there expires after the queue's retention period, so give the
   * dead-letter queue its own consumer.
   */
  deadLetterQueue?: string;
  /**
   * Consumer settings.
   */
  settings?: ConsumerSettings;
};

export interface ConsumerSettings {
  /**
   * The maximum number of messages per batch.
   * @default 10
   */
  batchSize?: number;
  /**
   * The maximum number of concurrent consumer invocations.
   */
  maxConcurrency?: number;
  /**
   * The maximum number of retries for a message.
   * @default 3
   */
  maxRetries?: number;
  /**
   * The maximum time to wait for a batch to fill, in milliseconds.
   * @default 5000
   */
  maxWaitTimeMs?: number;
  /**
   * The number of seconds to wait before retrying a message.
   */
  retryDelay?: number;
}

export type Consumer = Resource<
  "Cloudflare.Queues.Consumer",
  ConsumerProps,
  {
    consumerId: string;
    queueId: string;
    scriptName: string;
    accountId: string;
    deadLetterQueue?: string;
    settings?: ConsumerSettings;
    /**
     * Dev only, live queues only: the real queue's name, resolved from the
     * cloud so the local runtime can wire the broker + pull loop for a
     * locally-running consumer of an `Alchemy.remote()` queue.
     */
    queueName?: string;
    /**
     * Dev only, live queues only: id of the `http_pull` consumer attached
     * to the real queue so the local runtime can drain it via the HTTP
     * pull API. Deleted when this resource is deleted.
     */
    pullConsumerId?: string;
  },
  never,
  Providers
>;

/**
 * A Cloudflare Queue Consumer that processes messages from a Queue.
 *
 * Register a Worker as a consumer of a Queue. The Worker's `queue()`
 * handler will be invoked with batches of messages.
 *
 * Cloudflare allows at most one Worker consumer per queue (HTTP-pull
 * consumers can coexist). The reconciler enforces this: if the queue
 * already has a Worker consumer pointing at a different logical Worker's
 * script, the deploy fails with a clear error rather than silently
 * adopting it. A stranded consumer from a prior generation of the *same*
 * Worker (identified by the scripts' ownership tags) is rebuilt in place.
 * ### Registering a Consumer
 * **Example:** Basic consumer
 * ```typescript
 * const queue = yield* Cloudflare.Queues.Queue("MyQueue");
 * const worker = yield* Cloudflare.Worker("Worker", { ... });
 *
 * yield* Cloudflare.Queues.Consumer("MyConsumer", {
 *   queueId: queue.queueId,
 *   scriptName: worker.workerName,
 * });
 * ```
 *
 * **Example:** Consumer with settings
 * ```typescript
 * yield* Cloudflare.Queues.Consumer("MyConsumer", {
 *   queueId: queue.queueId,
 *   scriptName: worker.workerName,
 *   settings: {
 *     batchSize: 50,
 *     maxRetries: 5,
 *     maxWaitTimeMs: 10000,
 *   },
 * });
 * ```
 *
 * ### Dead-Lettering Failed Messages
 * **Example:** Consume the dead-letter queue
 * ```typescript
 * const dlq = yield* Cloudflare.Queues.Queue("MyDeadLetterQueue");
 *
 * yield* Cloudflare.Queues.Consumer("MyConsumer", {
 *   queueId: queue.queueId,
 *   scriptName: worker.workerName,
 *   deadLetterQueue: dlq.queueName,
 *   settings: { maxRetries: 3 },
 * });
 *
 * // Without a consumer, dead-lettered messages expire unseen.
 * yield* Cloudflare.Queues.Consumer("MyDeadLetterConsumer", {
 *   queueId: dlq.queueId,
 *   scriptName: worker.workerName,
 * });
 * ```
 *
 * **Example:** Dead-letter to a queue in another stack
 * ```typescript
 * const dlq = yield* Cloudflare.Queues.Queue.ref("DeadLetters", {
 *   stack: "shared",
 *   stage: "production",
 * });
 *
 * yield* Cloudflare.Queues.Consumer("MyConsumer", {
 *   queueId: queue.queueId,
 *   scriptName: worker.workerName,
 *   deadLetterQueue: dlq.queueName,
 * });
 * ```
 *
 * **Example:** Dead-letter to a queue by name
 * ```typescript
 * yield* Cloudflare.Queues.Consumer("MyConsumer", {
 *   queueId: queue.queueId,
 *   scriptName: worker.workerName,
 *   deadLetterQueue: "orders-dead-letters",
 * });
 * ```
 *
 * @resource
 * @product Queues
 * @category Storage & Databases
 */
export const Consumer = Resource<Consumer>("Cloudflare.Queues.Consumer", {
  aliases: ["Cloudflare.QueueConsumer"],
});

/**
 * Find and detach every worker queue-consumer pointing at `scriptName`.
 * Queue consumers have no by-script lookup, so scan the account's queues
 * (the list response inlines each queue's consumers). Waits until each
 * detach propagates to the workers subsystem so a follow-up deleteScript
 * doesn't re-race the conflict.
 *
 * Shared by the Worker provider's script delete (QueueConsumerConflict
 * recovery) and the Queue provider's orphaned-script cleanup.
 *
 * @internal
 */
export const detachQueueConsumersOfScript = Effect.fn(function* (
  accountId: string,
  scriptName: string,
) {
  const pages = yield* queues.listQueues.pages({ accountId }).pipe(
    Stream.runCollect,
    Effect.catchTag("InvalidRoute", () => Effect.succeed([])),
  );
  const targets = Array.from(pages).flatMap((page) =>
    (page.result ?? []).flatMap((queue) =>
      (queue.consumers ?? []).flatMap((consumer) =>
        queue.queueId &&
        consumer.type === "worker" &&
        consumer.consumerId &&
        "scriptName" in consumer &&
        consumer.scriptName === scriptName
          ? [{ queueId: queue.queueId, consumerId: consumer.consumerId }]
          : [],
      ),
    ),
  );
  yield* Effect.forEach(
    targets,
    ({ queueId, consumerId }) =>
      queues.deleteConsumer({ accountId, queueId, consumerId }).pipe(
        Effect.catchTag(["ConsumerNotFound", "QueueNotFound"], () => Effect.void),
        Effect.andThen(
          queues.getConsumer({ accountId, queueId, consumerId }).pipe(
            Effect.flatMap(() => Effect.fail("still-attached" as const)),
            Effect.catchTag(["ConsumerNotFound", "QueueNotFound"], () => Effect.void),
            Effect.retry({
              while: (e) => e === "still-attached",
              schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(30)]),
            }),
            Effect.ignore,
          ),
        ),
      ),
    { concurrency: 5, discard: true },
  );
});

// Cloudflare allows a single Worker consumer per queue, so the
// first match in the paginated stream is the only one. Using
// `.items` defeats single-page lookups that would otherwise
// miss late-arriving consumers under eventual consistency.
const findWorkerConsumer = (acct: string, queueId: string) =>
  queues.listConsumers.items({ accountId: acct, queueId }).pipe(
    Stream.map(toObserved),
    Stream.filter((c): c is ObservedConsumer => c !== undefined),
    Stream.runHead,
    Effect.map(Option.getOrUndefined),
    // A queue created earlier in the same plan is eventually consistent:
    // listConsumers can transiently 404 (`QueueNotFound`, code 11000) before
    // the fresh queue is visible to the consumers API. Treat that as "no
    // consumer yet" — the caller falls through to createConsumer, which
    // retries QueueNotFound until the queue propagates.
    Effect.catchTag("QueueNotFound", () => Effect.undefined),
  );

export const ConsumerProviderLive = () =>
  Provider.succeed(Consumer, {
    // The `consumerId` is not marked as stable because if you start in dev mode, the ID will change on first deploy.
    stables: ["accountId"],
    // Queue consumers are sub-resources of a queue with no account-wide
    // enumeration API, so fan out: list every queue in the account, then
    // list each queue's consumers and keep the worker ones (the only kind
    // this resource manages, matching `read`/`reconcile`).
    list: Effect.fn(function* () {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const queueIds = yield* queues.listQueues.pages({ accountId }).pipe(
        Stream.runCollect,
        Effect.map((chunk) =>
          Array.from(chunk).flatMap((page) =>
            (page.result ?? []).map((q) => q.queueId).filter((id): id is string => id != null),
          ),
        ),
        // Account not entitled for Queues — nothing to enumerate.
        Effect.catchTag("InvalidRoute", () => Effect.succeed<string[]>([])),
      );
      const rows = yield* Effect.forEach(
        queueIds,
        (queueId) =>
          queues.listConsumers.pages({ accountId, queueId }).pipe(
            Stream.runCollect,
            Effect.map((chunk) =>
              Array.from(chunk).flatMap((page) =>
                (page.result ?? []).flatMap((c): Consumer["Attributes"][] => {
                  if (c.type !== "worker" || !c.consumerId) return [];
                  const s = c.settings ?? undefined;
                  return [
                    {
                      consumerId: c.consumerId,
                      queueId,
                      scriptName: c.scriptName ?? "",
                      accountId,
                      deadLetterQueue: c.deadLetterQueue ?? undefined,
                      settings: s
                        ? {
                            batchSize: s.batchSize ?? undefined,
                            maxConcurrency: s.maxConcurrency ?? undefined,
                            maxRetries: s.maxRetries ?? undefined,
                            maxWaitTimeMs: s.maxWaitTimeMs ?? undefined,
                            retryDelay: s.retryDelay ?? undefined,
                          }
                        : undefined,
                    },
                  ];
                }),
              ),
            ),
            // Queue deleted mid-list or partial entitlement — skip it.
            Effect.catchTag(["QueueNotFound", "InvalidRoute"], () =>
              Effect.succeed<Consumer["Attributes"][]>([]),
            ),
          ),
        { concurrency: 10 },
      );
      return rows.flat();
    }),
    diff: Effect.fn(function* ({ olds, news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      if (!isResolved(news)) return undefined;
      // If either contain `dev:` IDs, we need to update to live ones.
      // The live resource doesn't exist yet, so there's no need to replace even when we otherwise would.
      if (!isLiveId(output?.queueId) || !isLiveId(output?.consumerId)) {
        return { action: "update" };
      }
      if ((output?.accountId ?? accountId) !== accountId) {
        return { action: "replace" } as const;
      }
      // Queue change requires replacement — consumerId is bound
      // to a queue and the API has no "move consumer" verb.
      if (output?.queueId && news.queueId !== output.queueId) {
        return { action: "replace", deleteFirst: true } as const;
      }
      // Settings / DLQ / script drift is an update. We DON'T
      // escalate scriptName changes to `replace` because the
      // engine resolves cross-resource Output<string> refs (a
      // sibling Worker's `workerName`) lazily — when the upstream
      // Worker is created in the same plan, `news` is partially
      // unresolved at diff time and `isResolved(news)` short-
      // circuits up top. Falling through to "update" lets the
      // engine call reconcile with fully-resolved `news`, where
      // we detect script drift and rebuild the consumer in
      // place (Cloudflare's PUT silently ignores `script_name`
      // changes, so reconcile does delete-then-create).
      if (
        JSON.stringify(olds.settings ?? {}) !== JSON.stringify(news.settings ?? {}) ||
        (olds.deadLetterQueue ?? undefined) !== (news.deadLetterQueue ?? undefined) ||
        (output?.scriptName !== undefined && news.scriptName !== output.scriptName)
      ) {
        return { action: "update" } as const;
      }
    }),
    reconcile: Effect.fn(function* ({ news, output }) {
      const { accountId } = yield* yield* CloudflareEnvironment;
      const acct = output?.accountId ?? accountId;
      // Only use `output.queueId` if it's a live ID (i.e. no `dev:` prefix).
      // Otherwise, the lookup will fail because the request is malformed.
      const queueId = isLiveId(output?.queueId) ? output.queueId : news.queueId;

      // Delete a consumer and wait for Cloudflare's worker subsystem to
      // drop its claim on the old script so createConsumer below doesn't
      // race the queue↔script propagation lag.
      const detachConsumer = Effect.fn(function* (consumerId: string) {
        yield* queues
          .deleteConsumer({ accountId: acct, queueId, consumerId })
          .pipe(Effect.catchTag("ConsumerNotFound", () => Effect.void));
        yield* queues.getConsumer({ accountId: acct, queueId, consumerId }).pipe(
          Effect.flatMap(() => Effect.fail("still-attached" as const)),
          Effect.catchTag("ConsumerNotFound", () => Effect.void),
          Effect.retry({
            while: (e) => e === "still-attached",
            schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(30)]),
          }),
          Effect.ignore,
        );
      });

      // Observe — prefer the cached consumerId, then fall back to
      // listConsumers (paginated) to recover from out-of-band
      // deletes or partial state-persistence failures. Track
      // whether the observation came from the cached id or the
      // list scan: a different-script worker consumer found via
      // the list scan is potentially foreign (state was lost,
      // someone else attached the consumer), and silently
      // updating it could clobber another team's wiring.
      let observed: ObservedConsumer | undefined;
      let owned = false;
      if (isLiveId(output?.consumerId)) {
        const fetched = yield* queues
          .getConsumer({ accountId: acct, queueId, consumerId: output.consumerId })
          .pipe(Effect.catchTag("ConsumerNotFound", () => Effect.undefined));
        if (fetched) {
          observed = toObserved(fetched);
          owned = observed !== undefined;
        }
      }
      if (!observed) {
        observed = yield* findWorkerConsumer(acct, queueId);
      }

      // Owned consumer pointing at a different script: rebuild
      // it in place. Cloudflare's PUT consumer silently ignores
      // `script_name` changes on existing consumers (the live
      // record stays pinned to the original worker), and the
      // platform allows only one Worker consumer per queue, so
      // the only path to re-point is delete-then-create.
      if (
        owned &&
        observed &&
        observed.scriptName !== undefined &&
        observed.scriptName !== news.scriptName
      ) {
        yield* detachConsumer(observed.consumerId);
        observed = undefined;
        owned = false;
      }

      // A consumer found via the list scan (state loss) pointing at a
      // different script. Probe the script it points at: when it no
      // longer exists, or is a *prior generation of the same logical
      // Worker* (same `alchemy:stack/stage/id` ownership tags as the
      // configured script — replacements mint a new physical name, and
      // a crashed apply can strand the old generation's consumer with
      // its state entry lost), the wiring is ours to rebuild. Any other
      // script — another team's, or a *different* Worker in this stack
      // (a real misconfiguration: two consumers declared for one queue)
      // — is refused loudly rather than silently stolen.
      if (
        observed &&
        !owned &&
        observed.scriptName !== undefined &&
        observed.scriptName !== news.scriptName
      ) {
        const scriptTags = Effect.fn(function* (scriptName: string) {
          const settings = yield* workers
            .getScriptScriptAndVersionSetting({ accountId: acct, scriptName })
            .pipe(
              Effect.catchTag(["WorkerNotFound", "WorkerHasNoVersions"], () => Effect.undefined),
            );
          return settings === undefined ? undefined : new Set(settings.tags ?? []);
        });
        const observedTags = yield* scriptTags(observed.scriptName);
        let staleGeneration = observedTags === undefined;
        if (observedTags !== undefined) {
          const stack = yield* Stack;
          const desiredTags = news.scriptName ? yield* scriptTags(news.scriptName) : undefined;
          const observedId = Array.from(observedTags).find((t) => t.startsWith("alchemy:id:"));
          staleGeneration =
            observedId !== undefined &&
            observedTags.has(`alchemy:stack:${stack.name}`) &&
            observedTags.has(`alchemy:stage:${stack.stage}`) &&
            desiredTags !== undefined &&
            desiredTags.has(observedId) &&
            desiredTags.has(`alchemy:stack:${stack.name}`) &&
            desiredTags.has(`alchemy:stage:${stack.stage}`);
        }
        if (!staleGeneration) {
          return yield* Effect.die(
            `Cloudflare queue "${queueId}" already has a worker ` +
              `consumer for script "${observed.scriptName}", but this ` +
              `resource is configured for "${news.scriptName}" and ` +
              `local state for the consumer was missing. Each queue ` +
              `can have only one worker consumer — delete the ` +
              `existing one, update scriptName to match, or restore ` +
              `the consumer's state entry before redeploying.`,
          );
        }
        yield* detachConsumer(observed.consumerId);
        observed = undefined;
      }

      // Ensure — create if missing. ConsumerAlreadyExists is the
      // race signal: another reconcile or peer beat us to it.
      // Re-run the lookup; the paginated stream tolerates the
      // single-page eventual-consistency window the previous
      // implementation missed.
      let consumerId: string;
      if (!observed) {
        const created = yield* queues
          .createConsumer({
            accountId: acct,
            queueId,
            scriptName: news.scriptName,
            type: "worker",
            deadLetterQueue: news.deadLetterQueue,
            settings: news.settings,
          })
          .pipe(
            // Two eventual-consistency races on a fresh deploy, both
            // retried on the same bounded schedule:
            //  - QueueHandlerMissing (11001): the sibling Worker resource
            //    pre-creates a placeholder script with no `queue` handler;
            //    Cloudflare returns 11001 until the real reconcile uploads
            //    the handler.
            //  - QueueNotFound (11000): the queue was created earlier in
            //    this same plan but isn't yet visible to the consumers API.
            Effect.tapError((e) =>
              e._tag === "QueueHandlerMissing" || e._tag === "QueueNotFound"
                ? Effect.logDebug(
                    `Consumer create: not ready for worker ` +
                      `"${news.scriptName}" (${e._tag}), retrying`,
                  )
                : Effect.void,
            ),
            Effect.retry({
              while: (e) => e._tag === "QueueHandlerMissing" || e._tag === "QueueNotFound",
              schedule: queueHandlerReadinessSchedule,
            }),
            Effect.catchTag("ConsumerAlreadyExists", (cause) =>
              Effect.gen(function* () {
                const match = yield* findWorkerConsumer(acct, queueId);
                if (!match) {
                  return yield* Effect.die(
                    `Cloudflare reported a worker consumer already ` +
                      `exists on queue "${queueId}", but listConsumers ` +
                      `returned none. Retry the deploy; if this ` +
                      `persists, the queue is in an inconsistent ` +
                      `state. Underlying error: ${cause.message}`,
                  );
                }
                if (match.scriptName !== undefined && match.scriptName !== news.scriptName) {
                  return yield* Effect.die(
                    `Cloudflare queue "${queueId}" already has a ` +
                      `worker consumer for script "${match.scriptName}", ` +
                      `but this resource is configured for ` +
                      `"${news.scriptName}". Each queue can have only ` +
                      `one worker consumer — delete the existing one ` +
                      `or update scriptName to match before redeploying.`,
                  );
                }
                return match;
              }),
            ),
          );
        consumerId = created.consumerId!;
      } else {
        consumerId = observed.consumerId;
      }

      // Sync — Cloudflare replaces all mutable fields on
      // updateConsumer, so always issue this so adoption converges
      // and settings drift gets corrected on every reconcile.
      // updateConsumer hits the same "queue handler missing" race
      // window as create when the worker is mid-upload, so apply
      // the same bounded retry.
      yield* queues
        .updateConsumer({
          accountId: acct,
          queueId,
          consumerId,
          scriptName: news.scriptName,
          type: "worker",
          settings: news.settings,
          deadLetterQueue: news.deadLetterQueue,
        })
        .pipe(
          Effect.retry({
            while: (e) => e._tag === "QueueHandlerMissing",
            schedule: queueHandlerReadinessSchedule,
          }),
        );

      yield* queues.getConsumer({ accountId: acct, queueId, consumerId }).pipe(
        Effect.flatMap((fetched) =>
          toObserved(fetched)?.scriptName === news.scriptName
            ? Effect.void
            : Effect.fail("ScriptUnbound" as const),
        ),
        Effect.catchTag("ConsumerNotFound", () => Effect.fail("ScriptUnbound" as const)),
        Effect.retry({
          while: (e) => e === "ScriptUnbound",
          schedule: queueHandlerReadinessSchedule,
        }),
      );

      return {
        consumerId,
        queueId,
        scriptName: news.scriptName!,
        accountId: acct,
        deadLetterQueue: news.deadLetterQueue,
        settings: news.settings,
      };
    }),
    delete: Effect.fn(function* ({ output }) {
      // If the consumerId is a `dev:` ID, the resource only exists locally, so we don't need to delete it from Cloudflare.
      if (!isLiveId(output.consumerId)) return;

      yield* queues
        .deleteConsumer({
          accountId: output.accountId,
          queueId: output.queueId,
          consumerId: output.consumerId,
        })
        .pipe(Effect.catchTag("ConsumerNotFound", () => Effect.void));

      // Block until Cloudflare's worker subsystem stops claiming
      // the script as a queue consumer. Without this the
      // sibling Worker.delete races on `QueueConsumerConflict`
      // (code 10064) — `deleteConsumer` returns success on the
      // queue subsystem before the script-side view propagates.
      yield* queues
        .getConsumer({
          accountId: output.accountId,
          queueId: output.queueId,
          consumerId: output.consumerId,
        })
        .pipe(
          Effect.flatMap(() => Effect.fail("still-attached" as const)),
          Effect.catchTag("ConsumerNotFound", () => Effect.void),
          Effect.retry({
            while: (e) => e === "still-attached",
            schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(30)]),
          }),
          Effect.ignore,
        );
    }),
    read: Effect.fn(function* ({ output }) {
      if (output?.consumerId) {
        const fetched = yield* queues
          .getConsumer({
            accountId: output.accountId,
            queueId: output.queueId,
            consumerId: output.consumerId,
          })
          .pipe(Effect.catchTag("ConsumerNotFound", () => Effect.undefined));
        if (fetched) {
          return {
            consumerId: fetched.consumerId!,
            queueId: output.queueId,
            scriptName: toObserved(fetched)?.scriptName ?? output.scriptName,
            accountId: output.accountId,
            deadLetterQueue: output.deadLetterQueue,
            settings: output.settings,
          };
        }
      }
      // Fallback: a state loss can leave us without a consumerId
      // even though the consumer is still alive on Cloudflare. The
      // queue allows only one worker consumer, so finding it via
      // listConsumers is unambiguous.
      if (output?.queueId && output?.accountId) {
        const match = yield* findWorkerConsumer(output.accountId, output.queueId);
        if (match) {
          return {
            consumerId: match.consumerId,
            queueId: output.queueId,
            scriptName: match.scriptName ?? output.scriptName,
            accountId: output.accountId,
            deadLetterQueue: output.deadLetterQueue,
            settings: output.settings,
          };
        }
      }
      return undefined;
    }),
  });

type ObservedConsumer = { consumerId: string; scriptName: string | undefined };

const toObserved = (c: {
  consumerId?: string | null;
  scriptName?: string | null;
  type?: "worker" | "http_pull" | null;
}): ObservedConsumer | undefined =>
  c.consumerId && c.type === "worker"
    ? { consumerId: c.consumerId, scriptName: c.scriptName ?? undefined }
    : undefined;

// ~60s budget — Worker reconcile uploads typically land in 2–10s,
// but a fresh container/asset deploy can stretch that.
const queueHandlerReadinessSchedule = Schedule.max([
  Schedule.spaced("2 seconds"),
  Schedule.recurs(30),
]);

export const ConsumerProviderLocal = () =>
  RpcProvider.effect(
    Consumer,
    LOCAL_PROVIDERS_URL,
    Effect.gen(function* () {
      const localRuntimeState = yield* LocalRuntimeState;

      // Restart the locally running workerd instances (if any) for the
      // given scripts so start-time queue-consumer wiring is re-read from
      // `localRuntimeState.queueConsumers`. See `workerRestarts` on
      // {@link LocalRuntimeState}.
      const restartScripts = (scriptNames: ReadonlyArray<string>) =>
        Effect.forEach(
          scriptNames,
          (scriptName) =>
            MutableHashMap.get(localRuntimeState.workerRestarts, scriptName).pipe(
              Option.match({ onNone: () => Effect.void, onSome: (restart) => restart }),
            ),
          { discard: true },
        );

      return {
        list: () =>
          Effect.sync(() => Array.from(MutableHashMap.values(localRuntimeState.queueConsumers))),
        diff: Effect.fn(function* ({ news, output }) {
          if (!output) return { action: "update" };
          // A real (non-`dev:`) consumerId on a local-mode row is legacy
          // damage from pre-stamping dev runs — replace so the new
          // generation mints a true local identity (delete best-effort
          // detaches the stray live consumer).
          if (isLiveId(output.consumerId)) {
            return { action: "replace" };
          }
          if (!isResolved(news)) return undefined;
          const accountId = isLiveId(news.queueId)
            ? (yield* yield* CloudflareEnvironment).accountId
            : yield* localAccountId;
          if (output.queueId !== news.queueId || output.accountId !== accountId) {
            return { action: "replace" };
          }
          if (
            JSON.stringify(output.settings ?? {}) !== JSON.stringify(news.settings ?? {}) ||
            output.scriptName !== news.scriptName ||
            output.deadLetterQueue !== news.deadLetterQueue
          ) {
            return { action: "update" };
          }
          // If the resource is a noop, add it to the local runtime state so it's available downstream.
          // We do it here instead of in the reconcile function so it doesn't appear as an update.
          MutableHashMap.set(localRuntimeState.queueConsumers, output.consumerId, output);
          return { action: "noop" };
        }),
        read: Effect.fn(function* ({ output }) {
          if (!output?.consumerId) return undefined;
          return MutableHashMap.get(localRuntimeState.queueConsumers, output.consumerId).pipe(
            Option.getOrUndefined,
          );
        }),
        reconcile: Effect.fn(function* ({ news, output }) {
          const accountId = isLiveId(news.queueId)
            ? (yield* yield* CloudflareEnvironment).accountId
            : yield* localAccountId;
          // A LIVE queue (`Alchemy.remote()`) consumed by a LOCAL worker:
          // Cloudflare only pushes to deployed consumers, so the local
          // runtime drains the real queue via the HTTP pull API instead.
          // Observe the queue (name + existing consumers) and ensure an
          // `http_pull` consumer exists for the pull loop to lease from.
          let queueName: string | undefined;
          let pullConsumerId: string | undefined;
          if (isLiveId(news.queueId)) {
            const queue = yield* queues.getQueue({ accountId, queueId: news.queueId });
            queueName = queue.queueName ?? undefined;
            const existingPull = (queue.consumers ?? []).find((c) => c.type === "http_pull");
            if (existingPull?.consumerId) {
              pullConsumerId = existingPull.consumerId;
            } else {
              const created = yield* queues.createConsumer({
                accountId,
                queueId: news.queueId,
                type: "http_pull",
                settings: {
                  batchSize: news.settings?.batchSize,
                  maxRetries: news.settings?.maxRetries,
                  retryDelay: news.settings?.retryDelay,
                },
              });
              pullConsumerId =
                ("consumerId" in created ? created.consumerId : undefined) ?? undefined;
            }
          }
          const consumer: Consumer["Attributes"] = {
            // Never carry a real (non-`dev:`) consumer id forward onto a
            // local row — it belongs to a live consumer this row no longer
            // manages (legacy pre-stamping damage).
            consumerId:
              output?.consumerId && !isLiveId(output.consumerId)
                ? output.consumerId
                : `dev:${crypto.randomUUID()}`,
            queueId: news.queueId,
            scriptName: news.scriptName,
            deadLetterQueue: news.deadLetterQueue,
            accountId,
            settings: news.settings,
            queueName,
            pullConsumerId,
          };
          MutableHashMap.set(localRuntimeState.queueConsumers, consumer.consumerId, consumer);
          // A local workerd instance only reads its queue-consumer wiring
          // at start time, and this reconcile races the sibling Worker's
          // start (the Worker's `precreate` resolves `scriptName` before
          // workerd is up). Restart any already-running instance so it
          // picks up the new wiring; if the worker hasn't served yet this
          // is a no-op and its first serve observes the map we just set.
          yield* restartScripts(
            output && output.scriptName !== consumer.scriptName
              ? [output.scriptName, consumer.scriptName]
              : [consumer.scriptName],
          );
          return consumer;
        }),
        delete: Effect.fn(function* ({ output }) {
          MutableHashMap.remove(localRuntimeState.queueConsumers, output.consumerId);
          yield* restartScripts([output.scriptName]);
          // Legacy local-mode rows written before providerMode stamping can
          // carry a real consumer's id — detach the live consumer too so
          // migrating the row to a true local identity doesn't strand it on
          // the queue (a stranded worker consumer blocks every future
          // createConsumer with ConsumerAlreadyExists).
          if (isLiveId(output.consumerId) && isLiveId(output.queueId)) {
            yield* queues
              .deleteConsumer({
                accountId: output.accountId,
                queueId: output.queueId,
                consumerId: output.consumerId,
              })
              .pipe(Effect.catchTag(["ConsumerNotFound", "QueueNotFound"], () => Effect.void));
          }
          // Remove the http_pull consumer this row attached to its live
          // queue. Idempotent: gone-already (or queue deleted first) is
          // success.
          if (output.pullConsumerId && isLiveId(output.queueId)) {
            yield* queues
              .deleteConsumer({
                accountId: output.accountId,
                queueId: output.queueId,
                consumerId: output.pullConsumerId,
              })
              .pipe(Effect.catchTag(["ConsumerNotFound", "QueueNotFound"], () => Effect.void));
          }
        }),
      };
    }),
  );

export const ConsumerProvider = () =>
  ProviderLayer.dual(Consumer, {
    local: () => ConsumerProviderLocal().pipe(Layer.provide(localRuntimeServices())),
    live: () => ConsumerProviderLive(),
  });