// Alchemy modifications are licensed under Apache-2.0.
// This file includes third-party code; see /THIRD_PARTY_LICENSES.md.
// Alchemy modifications: uses Array<T> syntax for non-tuple array types to match the repository convention.
import { DurableObject } from "cloudflare:workers";
import type { WorkflowEntrypoint, WorkflowEvent, WorkflowStep } from "cloudflare:workers";
import type { RestartFromStep, WorkflowInstanceTerminateOptions } from "./binding.ts";
import { Context, REDACTED_STEP_OUTPUT } from "./context.ts";
import type { Event } from "./context.ts";
import {
INSTANCE_METADATA,
InstanceEvent,
InstanceStatus,
instanceStatusName,
InstanceTrigger,
toInstanceStatus,
} from "./instance.ts";
import type { InstanceMetadata, RawInstanceLog } from "./instance.ts";
import { computeHash } from "./lib/cache.ts";
import {
ABORT_REASONS,
createWorkflowError,
isAbortError,
PreservedNonRetryableError,
shouldPreserveNonRetryableError,
stepNotFoundError,
WorkflowFatalError,
} from "./lib/errors.ts";
import {
ENGINE_TIMEOUT,
GracePeriodSemaphore,
startGracePeriod,
} from "./lib/gracePeriodSemaphore.ts";
import {
readAndClearRestartFromStep,
resolveGroupKeysToWipe,
storeRestartFromStep,
wipeRestartState,
} from "./lib/restart.ts";
import {
clearRollbackRegistry,
disposeRollbackStub,
executeRollbacks,
registerRollbackFn,
ROLLBACK_CACHE_KEY_PREFIX,
} from "./lib/rollback.ts";
import type { RollbackRegistration, RollbackRegistryEntry } from "./lib/rollback.ts";
import { normalizeForStorage } from "./lib/serialization.ts";
import {
createReplayReadableStream,
getInvalidStoredStreamOutputError,
getStoredStreamOutputPreview,
getStreamOutputMetaKey,
StreamOutputState,
} from "./lib/streams.ts";
import type { StreamOutputMeta } from "./lib/streams.ts";
import { TimePriorityQueue } from "./lib/timePriorityQueue.ts";
import { MODIFIER_KEYS, WorkflowInstanceModifier } from "./modifier.ts";
import {
isTerminalEvent,
parseResolvedStepConfig,
WorkflowSubscriptionTarget,
} from "./subscription.ts";
import type {
WorkflowSubscriptionEvent,
WorkflowSubscriptionOptions,
WorkflowSubscriptionState,
} from "./subscription.ts";
interface Env {
ENGINE: DurableObjectNamespace<Engine>;
USER_WORKFLOW: WorkflowEntrypoint;
WORKFLOW_NAME?: string;
STEP_LIMIT?: string; // JSON-encoded number from miniflare binding
STEP_LIMITS?: Record<string, number>;
}
/**
* Set by workerd when the Engine runs a Workflow exposed on `ctx.exports`. A
* single Engine class then serves every Workflow of a Worker, so these replace
* the per-Workflow `USER_WORKFLOW` and `WORKFLOW_NAME` bindings (which are then
* absent), and the step limit is looked up by name in `STEP_LIMITS`.
*/
interface EngineProps {
workflowClass?: WorkflowEntrypoint;
workflowName?: string;
}
export type DatabaseWorkflow = {
name: string;
id: string;
created_on: string;
modified_on: string;
script_name: string;
class_name: string | null;
triggered_on: string | null;
};
export type DatabaseVersion = {
id: string;
class_name: string;
created_on: string;
modified_on: string;
workflow_id: string;
mutable_pipeline_id: string;
};
export type DatabaseInstance = {
id: string;
created_on: string;
modified_on: string;
workflow_id: string;
version_id: string;
status: InstanceStatus;
started_on: string | null;
ended_on: string | null;
};
export type Log = {
event: InstanceEvent;
group: string | null;
target: string | null;
metadata: {
result: string;
payload: string;
error: { name: string; message: string };
};
};
export type EngineLogs = {
logs: Array<Log>;
};
const ENGINE_STATUS_KEY = "ENGINE_STATUS";
const WORKFLOW_OUTPUT_KEY = "WORKFLOW_OUTPUT";
const EVENT_MAP_PREFIX = "EVENT_MAP";
export const DEFAULT_STEP_LIMIT = 10_000;
const PAUSE_DATETIME = "PAUSE_DATETIME";
export type RollbackPhase = "replay" | "rollback";
/**
* JSON.stringify replacer that converts TypedArrays and ArrayBuffers to a
* human-readable description. Without this, JSON.stringify(Uint8Array) encodes
* each byte as a numeric key ({"0":1,"1":2,...}), producing a string ~10x larger
* than byteLength and causing SQLITE_TOOBIG for outputs above ~170 KB.
* The replacer is called recursively by JSON.stringify, so nested binary values
* inside objects or arrays are also handled.
*/
function binaryReplacer(_key: string, value: unknown): unknown {
if (typeof value === "bigint") {
return `[BigInt(${value})]`;
}
if (value instanceof ArrayBuffer) {
return `[ArrayBuffer(${value.byteLength} bytes)]`;
}
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
return `[${value.constructor.name}(${(value as ArrayBufferView).byteLength} bytes)]`;
}
return value;
}
function isStepSuccessEvent(event: InstanceEvent): boolean {
return event === InstanceEvent.STEP_SUCCESS || event === InstanceEvent.ROLLBACK_STEP_SUCCESS;
}
type EngineWorkflowSubscriptionEvent =
| WorkflowSubscriptionEvent
| (Pick<WorkflowSubscriptionEvent, "instanceId" | "eventId" | "timestamp"> & {
type: "internal";
});
async function readStepConfig(storage: DurableObjectStorage, groupKey: string | null) {
if (groupKey === null) {
return undefined;
}
return parseResolvedStepConfig(await storage.get(`${groupKey}-config`));
}
async function readStepCompletedOutput(storage: DurableObjectStorage, groupKey: string | null) {
if (groupKey === null) {
return undefined;
}
const valueKey = `${groupKey}-value`;
const streamMetaKey = getStreamOutputMetaKey(groupKey);
const configKey = `${groupKey}-config`;
const stored = await storage.get([valueKey, streamMetaKey, configKey]);
const config = parseResolvedStepConfig(stored.get(configKey));
if (config === undefined) {
return undefined;
}
if (config.sensitive === "output") {
return REDACTED_STEP_OUTPUT;
}
const streamMeta = stored.get(streamMetaKey) as StreamOutputMeta | undefined;
if (streamMeta?.state === StreamOutputState.Complete) {
const integrityError = getInvalidStoredStreamOutputError(storage, groupKey, streamMeta);
if (integrityError !== undefined) {
throw createWorkflowError(
"Step has completed but its stored stream output is corrupt or incomplete",
"instance.step_output_corrupt",
);
}
return createReplayReadableStream({
storage,
cacheKey: groupKey,
meta: streamMeta,
});
}
return (stored.get(valueKey) as { value: unknown } | undefined)?.value;
}
async function buildWorkflowSubscriptionEvent(
storage: DurableObjectStorage,
log: RawInstanceLog,
instanceId: string,
params: unknown,
): Promise<EngineWorkflowSubscriptionEvent> {
const common = {
instanceId,
eventId: log.id,
timestamp: new Date(log.timestamp).valueOf(),
};
const parseMetadata = () =>
JSON.parse(log.metadata) as {
result?: unknown;
error: { name: string; message: string };
attempt: number;
retryDelayMs?: number;
durationMs: number;
event: string;
};
const stepEvent = (
createEvent: (stepName: string) => Extract<WorkflowSubscriptionEvent, { stepName: string }>,
): EngineWorkflowSubscriptionEvent =>
log.target === null ? { ...common, type: "internal" } : createEvent(log.target);
switch (log.event) {
case InstanceEvent.WORKFLOW_QUEUED:
return { ...common, type: "workflow_queued" };
case InstanceEvent.WORKFLOW_START:
return { ...common, type: "workflow_started", params };
case InstanceEvent.WORKFLOW_RUNNING:
return { ...common, type: "workflow_running" };
case InstanceEvent.WORKFLOW_PAUSED:
return { ...common, type: "workflow_paused" };
case InstanceEvent.WORKFLOW_WAITING_FOR_PAUSE:
return { ...common, type: "workflow_waiting_for_pause" };
case InstanceEvent.WORKFLOW_WAITING:
return { ...common, type: "workflow_waiting" };
case InstanceEvent.WORKFLOW_SUCCESS: {
const storedOutput = await storage.get<{ value: unknown }>(WORKFLOW_OUTPUT_KEY);
return {
...common,
type: "workflow_completed",
output: storedOutput === undefined ? parseMetadata().result : storedOutput.value,
};
}
case InstanceEvent.WORKFLOW_FAILURE:
return {
...common,
type: "workflow_errored",
error: parseMetadata().error,
};
case InstanceEvent.WORKFLOW_TERMINATED:
return { ...common, type: "workflow_terminated" };
case InstanceEvent.STEP_START: {
const config = await readStepConfig(storage, log.groupKey);
return stepEvent((stepName) => ({
...common,
type: "step_started",
stepName,
...(config === undefined ? {} : { config }),
}));
}
case InstanceEvent.STEP_SUCCESS: {
const output = await readStepCompletedOutput(storage, log.groupKey);
return stepEvent((stepName) => ({
...common,
type: "step_completed",
stepName,
...(output === undefined ? {} : { output }),
}));
}
case InstanceEvent.STEP_FAILURE:
return stepEvent((stepName) => ({
...common,
type: "step_errored",
stepName,
}));
case InstanceEvent.ATTEMPT_START:
return stepEvent((stepName) => ({
...common,
type: "attempt_started",
stepName,
attempt: parseMetadata().attempt,
}));
case InstanceEvent.ATTEMPT_SUCCESS:
return stepEvent((stepName) => ({
...common,
type: "attempt_completed",
stepName,
attempt: parseMetadata().attempt,
}));
case InstanceEvent.ATTEMPT_FAILURE: {
const metadata = parseMetadata();
return stepEvent((stepName) => ({
...common,
type: "attempt_errored",
stepName,
attempt: metadata.attempt,
error: metadata.error,
...(metadata.retryDelayMs === undefined ? {} : { retryDelayMs: metadata.retryDelayMs }),
}));
}
case InstanceEvent.SLEEP_START:
return stepEvent((stepName) => ({
...common,
type: "sleep_started",
stepName,
durationMs: parseMetadata().durationMs,
}));
case InstanceEvent.SLEEP_COMPLETE:
return stepEvent((stepName) => ({
...common,
type: "sleep_completed",
stepName,
}));
case InstanceEvent.WAIT_START:
return stepEvent((stepName) => ({
...common,
type: "wait_started",
stepName,
eventType: parseMetadata().event,
}));
case InstanceEvent.WAIT_COMPLETE:
return stepEvent((stepName) => ({
...common,
type: "wait_completed",
stepName,
}));
case InstanceEvent.WAIT_TIMED_OUT:
return stepEvent((stepName) => ({
...common,
type: "wait_timed_out",
stepName,
}));
case InstanceEvent.ROLLBACK_START:
return { ...common, type: "rollback_started" };
case InstanceEvent.ROLLBACK_STEP_START: {
const config = await readStepConfig(storage, log.groupKey);
return stepEvent((stepName) => ({
...common,
type: "rollback_step_started",
stepName,
...(config === undefined ? {} : { config }),
}));
}
case InstanceEvent.ROLLBACK_STEP_SUCCESS:
return stepEvent((stepName) => ({
...common,
type: "rollback_step_completed",
stepName,
}));
case InstanceEvent.ROLLBACK_STEP_FAILURE:
return stepEvent((stepName) => ({
...common,
type: "rollback_step_errored",
stepName,
error: parseMetadata().error,
}));
case InstanceEvent.ROLLBACK_ATTEMPT_START:
return stepEvent((stepName) => ({
...common,
type: "rollback_attempt_started",
stepName,
attempt: parseMetadata().attempt,
}));
case InstanceEvent.ROLLBACK_ATTEMPT_SUCCESS:
return stepEvent((stepName) => ({
...common,
type: "rollback_attempt_completed",
stepName,
attempt: parseMetadata().attempt,
}));
case InstanceEvent.ROLLBACK_ATTEMPT_FAILURE: {
const metadata = parseMetadata();
return stepEvent((stepName) => ({
...common,
type: "rollback_attempt_errored",
stepName,
attempt: metadata.attempt,
error: metadata.error,
...(metadata.retryDelayMs === undefined ? {} : { retryDelayMs: metadata.retryDelayMs }),
}));
}
case InstanceEvent.ROLLBACK_COMPLETE:
return { ...common, type: "rollback_completed" };
case InstanceEvent.ROLLBACK_FAILED:
return { ...common, type: "rollback_errored" };
case InstanceEvent.__INTERNAL_PROD:
return { ...common, type: "internal" };
}
}
export class Engine extends DurableObject<Env, EngineProps> {
logs: Array<unknown> = [];
isRunning: boolean = false;
accountId: number | undefined;
instanceId: string | undefined;
workflowName: string | undefined;
timeoutHandler: GracePeriodSemaphore;
priorityQueue: TimePriorityQueue | undefined;
stepLimit: number;
engineAbortController: AbortController = new AbortController();
pauseController: AbortController = new AbortController();
rollbackPhase: RollbackPhase | undefined = undefined;
rollbackEligibleCacheKeys: Set<string> | undefined = undefined;
waiters: Map<
string,
Array<[cacheKey: string, resolve: (event: Event | PromiseLike<Event>) => void]>
> = new Map();
eventMap: Map<string, Array<Event>> = new Map();
// Not persisted: rollback fns are RPC stubs, dead across DO restarts.
rollbackRegistry: Map<string, RollbackRegistryEntry> = new Map();
subscribers = new Set<WorkflowSubscriptionState>();
constructor(state: DurableObjectState, env: Env) {
super(state, env);
const { workflowName } = this.ctx.props;
this.stepLimit =
(workflowName === undefined ? undefined : env.STEP_LIMITS?.[workflowName]) ??
(env.STEP_LIMIT ? JSON.parse(env.STEP_LIMIT) : DEFAULT_STEP_LIMIT);
void this.ctx.blockConcurrencyWhile(async () => {
this.ctx.storage.transactionSync(() => {
try {
this.ctx.storage.sql.exec(`
CREATE TABLE IF NOT EXISTS priority_queue (
id INTEGER PRIMARY KEY NOT NULL,
created_on TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
target_timestamp INTEGER NOT NULL,
action INTEGER NOT NULL, -- should only be 0 or 1 (1 for added, 0 for deleted),
entryType INTEGER NOT NULL,
hash TEXT NOT NULL,
CHECK (action IN (0, 1)), -- guararentee that action can only be 0 or 1
UNIQUE (action, entryType, hash)
);
CREATE TABLE IF NOT EXISTS states (
id INTEGER PRIMARY KEY NOT NULL,
timestamp TIMESTAMP DEFAULT (DATETIME('now','subsec')),
groupKey TEXT,
target TEXT,
metadata TEXT,
event INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS streaming_step_chunks (
cache_key TEXT NOT NULL,
attempt INTEGER NOT NULL,
chunk_index INTEGER NOT NULL,
chunk BLOB NOT NULL,
PRIMARY KEY (cache_key, attempt, chunk_index)
) WITHOUT ROWID
`);
} catch (e) {
console.error(e);
throw e;
}
});
});
this.timeoutHandler = new GracePeriodSemaphore(startGracePeriod, ENGINE_TIMEOUT);
}
writeLog(
event: InstanceEvent,
group: string | null,
target: string | null = null,
metadata: Record<string, unknown>,
) {
this.ctx.storage.sql.exec(
"INSERT INTO states (event, groupKey, target, metadata) VALUES (?, ?, ?, ?)",
event,
group,
target,
JSON.stringify(metadata, binaryReplacer),
);
for (const subscriber of this.subscribers) {
const waiter = subscriber.waiter;
if (waiter !== undefined) {
subscriber.waiter = undefined;
waiter.resolve();
}
}
// Wake any waiters if this is a terminal step event
if (group) {
this.handleStepResultWaiter(group, event, metadata);
}
}
readEligibleRollbackStepsDesc(limit?: number): Array<{ cacheKey: string; target: string }> {
const rollbackTerminalGroups = new Set<string>();
const rollbackEligibleGroups = new Set<string>();
const stepStartsDesc: Array<{ groupKey: string; target: string | null }> = [];
const rows = [
...this.ctx.storage.sql.exec<{
event: InstanceEvent;
groupKey: string;
target: string | null;
metadata: string;
}>(
"SELECT event, groupKey, target, metadata FROM states WHERE groupKey IS NOT NULL ORDER BY id DESC",
),
];
for (const row of rows) {
if (row.event === InstanceEvent.STEP_START) {
stepStartsDesc.push({ groupKey: row.groupKey, target: row.target });
}
if (
row.event === InstanceEvent.ROLLBACK_STEP_SUCCESS ||
row.event === InstanceEvent.ROLLBACK_STEP_FAILURE
) {
rollbackTerminalGroups.add(
row.groupKey.startsWith(ROLLBACK_CACHE_KEY_PREFIX)
? row.groupKey.slice(ROLLBACK_CACHE_KEY_PREFIX.length)
: row.groupKey,
);
continue;
}
if (
row.event !== InstanceEvent.STEP_START &&
row.event !== InstanceEvent.STEP_SUCCESS &&
row.event !== InstanceEvent.STEP_FAILURE
) {
continue;
}
try {
if ((JSON.parse(row.metadata) as { hasRollback?: boolean }).hasRollback === true) {
rollbackEligibleGroups.add(row.groupKey);
}
} catch {
// Ignore malformed metadata in local persisted logs.
}
}
const eligible: Array<{ cacheKey: string; target: string }> = [];
for (const { groupKey, target } of stepStartsDesc) {
if (rollbackEligibleGroups.has(groupKey) && !rollbackTerminalGroups.has(groupKey)) {
eligible.push({ cacheKey: groupKey, target: target ?? groupKey });
if (limit !== undefined && eligible.length >= limit) {
break;
}
}
}
return eligible;
}
private getEligibleRollbackSteps(limit?: number): Array<string> {
return this.readEligibleRollbackStepsDesc(limit).map(({ cacheKey }) => cacheKey);
}
registerRollbackFn(registration: RollbackRegistration): void {
if (
this.rollbackPhase === "replay" &&
this.rollbackEligibleCacheKeys !== undefined &&
!this.rollbackEligibleCacheKeys.has(registration.cacheKey)
) {
disposeRollbackStub(registration.fn);
return;
}
registerRollbackFn(this.rollbackRegistry, registration);
}
setRollbackPhase(phase: RollbackPhase | undefined): void {
this.rollbackPhase = phase;
}
// Lives here for access to the protected DurableObject `ctx`.
createRollbackContext(rollbackStep?: { cacheKey: string }): Context {
return new Context(this, this.ctx, rollbackStep);
}
readLogsFromStep(_cacheKey: string): Array<RawInstanceLog> {
return [];
}
async subscribe(options?: WorkflowSubscriptionOptions): Promise<WorkflowSubscriptionTarget> {
const { cursor, filter } = options ?? {};
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
const state: WorkflowSubscriptionState = {
instanceId: metadata.instance.id,
params: metadata.event.payload,
lastEventId: cursor ?? -1,
filter: filter === undefined ? undefined : new Set(filter),
waiter: undefined,
closed: false,
};
const subscription = new WorkflowSubscriptionTarget(
() => this.nextWorkflowEvent(state),
() => {
state.closed = true;
this.subscribers.delete(state);
state.waiter?.resolve();
state.waiter = undefined;
},
);
this.subscribers.add(state);
return subscription;
}
private async nextWorkflowEvent(
state: WorkflowSubscriptionState,
): Promise<IteratorResult<WorkflowSubscriptionEvent, undefined>> {
while (!state.closed) {
const row = this.ctx.storage.sql
.exec<RawInstanceLog>(
"SELECT id, timestamp, event, groupKey, target, metadata FROM states WHERE id > ? ORDER BY id ASC LIMIT 1",
state.lastEventId,
)
.toArray()[0];
if (row === undefined) {
const hasTerminalEvent =
this.ctx.storage.sql
.exec<{ id: number }>(
"SELECT id FROM states WHERE id <= ? AND event IN (?, ?, ?) LIMIT 1",
state.lastEventId,
InstanceEvent.WORKFLOW_SUCCESS,
InstanceEvent.WORKFLOW_FAILURE,
InstanceEvent.WORKFLOW_TERMINATED,
)
.toArray()[0] !== undefined;
if (hasTerminalEvent) {
state.closed = true;
return { done: true, value: undefined };
}
await new Promise<void>((resolve) => {
state.waiter = { resolve };
});
continue;
}
state.lastEventId = row.id;
const event = await buildWorkflowSubscriptionEvent(
this.ctx.storage,
row,
state.instanceId,
state.params,
);
if (event.type === "internal") {
continue;
}
if (state.filter === undefined || state.filter.has(event.type)) {
return { done: false, value: event };
}
if (isTerminalEvent(event)) {
return { done: true, value: undefined };
}
}
return { done: true, value: undefined };
}
readLogs(): EngineLogs {
const logs = [
...this.ctx.storage.sql.exec<{
event: InstanceEvent;
groupKey: string | null;
target: string | null;
metadata: string;
}>("SELECT event, groupKey, target, metadata FROM states"),
];
return {
logs: logs.map((log) => {
const metadata = JSON.parse(log.metadata);
if (!isStepSuccessEvent(log.event) || !metadata.streamOutput) {
return { ...log, metadata, group: log.groupKey };
}
const { cacheKey, meta } = metadata.streamOutput as {
cacheKey: string;
meta: StreamOutputMeta;
};
try {
const preview = getStoredStreamOutputPreview({
storage: this.ctx.storage,
cacheKey,
meta,
maxChars: 1024,
});
metadata.result =
preview.type === "text"
? preview.output
: `[ReadableStream (binary): ${meta.totalBytes} bytes]`;
} catch {
metadata.result = `[ReadableStream: ${meta.totalBytes} bytes]`;
}
delete metadata.streamOutput;
return { ...log, metadata, group: log.groupKey };
}),
};
}
/**
* Returns detailed logs including timestamps, ordered by ID.
* Used by the local explorer to reconstruct step-level detail.
*/
readDetailedLogs(): Array<{
id: number;
timestamp: string;
event: number;
group: string | null;
target: string | null;
metadata: Record<string, unknown>;
}> {
const rows = [
...this.ctx.storage.sql.exec<{
id: number;
timestamp: string;
event: number;
groupKey: string | null;
target: string | null;
metadata: string;
}>("SELECT id, timestamp, event, groupKey, target, metadata FROM states ORDER BY id ASC"),
];
return rows.map((row) => {
const metadata = JSON.parse(row.metadata) as Record<string, unknown>;
if (!isStepSuccessEvent(row.event) || !metadata.streamOutput) {
return {
id: row.id,
timestamp: String(row.timestamp).replace(" ", "T") + "Z",
event: row.event,
group: row.groupKey,
target: row.target,
metadata,
};
}
const { cacheKey, meta } = metadata.streamOutput as {
cacheKey: string;
meta: StreamOutputMeta;
};
try {
const preview = getStoredStreamOutputPreview({
storage: this.ctx.storage,
cacheKey,
meta,
maxChars: 1024,
});
metadata.result =
preview.type === "text"
? preview.output
: `[ReadableStream (binary): ${meta.totalBytes} bytes]`;
} catch {
metadata.result = `[ReadableStream: ${meta.totalBytes} bytes]`;
}
delete metadata.streamOutput;
return {
id: row.id,
timestamp: String(row.timestamp).replace(" ", "T") + "Z",
event: row.event,
group: row.groupKey,
target: row.target,
metadata,
};
});
}
readLogsFromEvent(eventType: InstanceEvent): EngineLogs {
const logs = [
...this.ctx.storage.sql.exec<{
event: InstanceEvent;
groupKey: string | null;
target: string | null;
metadata: string;
}>("SELECT event, groupKey, target, metadata FROM states WHERE event = ?", eventType),
];
return {
logs: logs.map((log) => ({
...log,
metadata: JSON.parse(log.metadata),
group: log.groupKey,
})),
};
}
async getStatus(): Promise<InstanceStatus> {
if (this.accountId === undefined) {
// Engine could have restarted, so we try to restore from its state
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
// metadata was never set, so we assume the engine was never started
throw new Error("Engine was never started");
}
this.accountId = metadata.accountId;
this.instanceId = metadata.instance.id;
this.workflowName = metadata.workflow.name;
}
const res = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
// NOTE(lduarte): if status don't exist, means that engine is running for the first time, so we assume queued
if (res === undefined) {
return InstanceStatus.Queued;
}
return res;
}
// Returns instance metadata for the local explorer.
async getInstanceMetadata(): Promise<{
instanceId: string;
status: InstanceStatus;
createdOn: string;
}> {
const status = await this.getStatus();
// Read the full metadata to get the created_on timestamp
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
let createdOn = metadata?.instance?.created_on ?? "";
// Backfill for instances created before created_on was populated:
// try the earliest log entry, then the earliest priority queue entry.
if (!createdOn) {
const queries = [
"SELECT timestamp AS ts FROM states ORDER BY id ASC LIMIT 1",
"SELECT created_on AS ts FROM priority_queue ORDER BY id ASC LIMIT 1",
];
for (const query of queries) {
if (createdOn) {
break;
}
try {
for (const row of this.ctx.storage.sql.exec(query)) {
if (typeof row.ts === "string" && row.ts) {
// SQLite DATETIME('now','subsec') returns "YYYY-MM-DD HH:MM:SS.sss"
// which is not valid ISO 8601 (needs T separator and Z timezone).
const raw = row.ts.replace(" ", "T") + "Z";
createdOn = raw;
}
}
} catch {
// Table may not exist
}
}
}
return {
instanceId: this.instanceId ?? "",
status,
createdOn,
};
}
async setStatus(accountId: number, instanceId: string, status: InstanceStatus): Promise<void> {
const previousStatus = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
await this.ctx.storage.put(ENGINE_STATUS_KEY, status);
if (previousStatus !== status) {
switch (status) {
case InstanceStatus.Queued: {
// Creation writes WORKFLOW_QUEUED before the first status is stored.
const hasQueuedEvent =
this.ctx.storage.sql
.exec<{ id: number }>(
"SELECT id FROM states WHERE event = ? LIMIT 1",
InstanceEvent.WORKFLOW_QUEUED,
)
.toArray()[0] !== undefined;
if (!(previousStatus === undefined && hasQueuedEvent)) {
this.writeLog(InstanceEvent.WORKFLOW_QUEUED, null, null, {});
}
break;
}
case InstanceStatus.Running:
this.writeLog(InstanceEvent.WORKFLOW_RUNNING, null, null, {});
break;
case InstanceStatus.Paused:
this.writeLog(InstanceEvent.WORKFLOW_PAUSED, null, null, {});
break;
case InstanceStatus.WaitingForPause:
this.writeLog(InstanceEvent.WORKFLOW_WAITING_FOR_PAUSE, null, null, {});
break;
case InstanceStatus.Waiting:
this.writeLog(InstanceEvent.WORKFLOW_WAITING, null, null, {});
break;
}
}
// check if anyone is waiting for this status
this.handleStatusWaiter(status);
}
private statusWaiters: Map<
InstanceStatus,
{ resolve: () => void; reject: (e: unknown) => void }
> = new Map();
async waitForStatus(status: string): Promise<void> {
const targetStatus = toInstanceStatus(status);
const currentStatus = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
// if the workflow has already reached the desired state, resolve immediately
if (currentStatus === targetStatus) {
return;
}
// if it hasn't reached the desired state, create a new promise and add its resolver to the waiters map
return new Promise((resolve, reject) => {
this.statusWaiters.set(targetStatus, { resolve, reject });
// immediately reconcile against current status in case it's already finite
this.handleStatusWaiter(currentStatus as InstanceStatus);
});
}
handleStatusWaiter(status: InstanceStatus): void {
const waiter = this.statusWaiters.get(status);
// resolve if it reached the desired status
if (waiter) {
waiter.resolve();
this.statusWaiters.delete(status);
return;
}
switch (status) {
case InstanceStatus.Errored: {
// if it reaches final status "errored", then it can't be waiting for it to complete or terminate
const unreachableStatuses = [InstanceStatus.Complete, InstanceStatus.Terminated];
this.rejectUnreachableStatus(status, unreachableStatuses);
break;
}
case InstanceStatus.Terminated: {
// if it reaches final status "terminated", then it can't be waiting for it to complete or error
const unreachableStatuses = [InstanceStatus.Complete, InstanceStatus.Errored];
this.rejectUnreachableStatus(status, unreachableStatuses);
break;
}
case InstanceStatus.Complete: {
// if it reaches final status "complete", then it can't be waiting for it to terminate or error
const unreachableStatuses = [InstanceStatus.Terminated, InstanceStatus.Errored];
this.rejectUnreachableStatus(status, unreachableStatuses);
break;
}
default:
break;
}
}
rejectUnreachableStatus(reachedStatus: number, unreachableStatuses: Array<number>): void {
if (unreachableStatuses) {
for (const unreachableStatus of unreachableStatuses) {
const waiter = this.statusWaiters.get(unreachableStatus);
if (waiter) {
waiter.reject(
new Error(
`[WorkflowIntrospector] The Workflow instance ${
this.instanceId
} has reached status '${instanceStatusName(
reachedStatus,
)}'. This is a finite status that prevents it from ever reaching the expected status of '${instanceStatusName(
unreachableStatus,
)}'.`,
),
);
this.statusWaiters.delete(unreachableStatus);
return;
}
}
}
}
/**
* Create a replay ReadableStream from stored stream output metadata.
* Returns undefined if the stream data is not in a valid/complete state.
*/
private replayStreamFromMeta(streamOutput: {
cacheKey: string;
meta: StreamOutputMeta;
}): ReadableStream<Uint8Array> | undefined {
if (streamOutput.meta.state !== StreamOutputState.Complete) {
return undefined;
}
const integrityError = getInvalidStoredStreamOutputError(
this.ctx.storage,
streamOutput.cacheKey,
streamOutput.meta,
);
if (integrityError !== undefined) {
return undefined;
}
return createReplayReadableStream({
storage: this.ctx.storage,
cacheKey: streamOutput.cacheKey,
meta: streamOutput.meta,
});
}
private stepResultWaiters: Map<
string,
{ resolve: (v: unknown) => void; reject: (e: unknown) => void }
> = new Map();
async waitForStepResult(stepName: string, stepCount?: number): Promise<unknown> {
const hash = await computeHash(stepName);
const count = stepCount ?? 1;
const cacheKey = `${hash}-${count}`;
// read latest log from step
const rows = [
...this.ctx.storage.sql.exec<{
event: InstanceEvent;
metadata: string;
}>(
"SELECT event, metadata FROM states WHERE groupKey = ? ORDER BY id DESC LIMIT 1",
cacheKey,
),
];
if (rows.length > 0) {
const { event, metadata } = rows[0];
const parsed = JSON.parse(metadata);
if (event === InstanceEvent.STEP_SUCCESS) {
if (parsed?.streamOutput) {
return this.replayStreamFromMeta(parsed.streamOutput);
}
return parsed?.result;
}
if (event === InstanceEvent.STEP_FAILURE) {
throw parsed?.error ?? parsed;
}
}
// if it hasn't completed the step, create a new promise to later resolve/reject
return new Promise<unknown>((resolve, reject) => {
this.stepResultWaiters.set(cacheKey, { resolve, reject });
});
}
handleStepResultWaiter(group: string, event: InstanceEvent, metadata: Record<string, unknown>) {
const waiter = this.stepResultWaiters.get(group);
if (!waiter) {
return;
}
if (event === InstanceEvent.STEP_SUCCESS) {
if (metadata?.streamOutput) {
waiter.resolve(
this.replayStreamFromMeta(
metadata.streamOutput as {
cacheKey: string;
meta: StreamOutputMeta;
},
),
);
} else {
waiter.resolve(metadata?.result);
}
this.stepResultWaiters.delete(group);
} else if (event === InstanceEvent.STEP_FAILURE) {
const error = metadata?.error ?? new Error("Step failed");
waiter.reject(error);
this.stepResultWaiters.delete(group);
}
}
async getOutputOrError(isOutput: boolean): Promise<unknown> {
const status = await this.getStatus();
if (isOutput) {
if (status !== InstanceStatus.Complete) {
throw new Error(
`Cannot retrieve output: Workflow instance is in status "${instanceStatusName(
status,
)}" but must be "complete" to have an output available`,
);
}
const logs = this.readLogsFromEvent(InstanceEvent.WORKFLOW_SUCCESS).logs;
return logs.at(0)?.metadata.result;
} else {
if (status !== InstanceStatus.Errored) {
throw new Error(
`Cannot retrieve error: Workflow instance is in status "${instanceStatusName(
status,
)}" but must be "errored" to have error information available`,
);
}
const logs = this.readLogsFromEvent(InstanceEvent.WORKFLOW_FAILURE).logs;
const log = logs.at(0);
if (!log?.metadata.error) {
throw new Error("Cannot retrieve error: No workflow instance failure log found");
}
return log.metadata.error;
}
}
async abort(reason: string) {
await this.ctx.storage.sync();
// Clean up pending JS operations
this.timeoutHandler.dispose();
this.engineAbortController.abort(new Error(reason));
this.ctx.abort(reason);
}
// Called by the dispose function when introspecting the instance in tests
// TODO: Ideally this abort should be done by `abortAllDurableObjects` from worked called by vitest-plugin
async unsafeAbort(reason?: string) {
await this.ctx.storage.sync();
await this.ctx.storage.deleteAll();
this.ctx.abort(reason);
}
async storeEventMap() {
await this.ctx.blockConcurrencyWhile(async () => {
const entries: Record<string, Event> = {};
for (const [type, events] of this.eventMap.entries()) {
for (let i = 0; i < events.length; i++) {
entries[`${EVENT_MAP_PREFIX}\n${type}\n${i}`] = events[i];
}
}
if (Object.keys(entries).length > 0) {
await this.ctx.storage.put(entries);
}
});
}
async restoreEventMap() {
await this.ctx.blockConcurrencyWhile(async () => {
// FIXME(lduarte): can this OoM the DO in the production?
const entries = await this.ctx.storage.list<Event>({
prefix: EVENT_MAP_PREFIX,
});
for (const [key, value] of entries) {
const [_, eventType, _idx] = key.split("\n");
// NOTE(lduarte): safe to do because list returns keys in ascending order, so
// indexes will be correctly ordered
const eventList = this.eventMap.get(eventType) ?? [];
eventList.push(value);
this.eventMap.set(eventType, eventList);
}
});
}
async receiveEvent(event: { timestamp: Date; payload: unknown; type: string }) {
// There are four possible cases here:
// - There is a callback waiting, send it
// - There is no callback waiting but engine is alive, store it
// - Engine is not awake and is in Waiting status, store it and start it up
// - Engine is not awake and is in Paused (or another terminal) status, store it
// - Engine is not awake and is Errored or Terminated, this should not get called
let eventTypeQueue = this.eventMap.get(event.type) ?? [];
eventTypeQueue.push(event as Event);
this.eventMap.set(event.type, eventTypeQueue);
await this.storeEventMap();
// if the engine is running
if (this.isRunning) {
// Attempt to get the callback and run it
const callbacks = this.waiters.get(event.type);
if (callbacks) {
const entry = callbacks[0];
if (entry) {
const [, resolve] = entry;
resolve(event);
// Remove it from the list of callbacks
callbacks.shift();
this.waiters.set(event.type, callbacks);
eventTypeQueue = this.eventMap.get(event.type) ?? [];
eventTypeQueue.shift();
this.eventMap.set(event.type, eventTypeQueue);
return;
}
}
} else {
const mockEvent = await this.ctx.storage.get(`${MODIFIER_KEYS.MOCK_EVENT}${event.type}`);
if (mockEvent) {
return;
}
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw new Error("Engine was never started");
}
void this.init(
metadata.accountId,
metadata.workflow,
metadata.version,
metadata.instance,
metadata.event,
);
}
}
getInstanceModifier(): WorkflowInstanceModifier {
return new WorkflowInstanceModifier(this, this.ctx);
}
async changeInstanceStatus(
newStatus: "resume" | "pause" | "terminate" | "restart",
from?: RestartFromStep,
terminateOptions?: WorkflowInstanceTerminateOptions,
): Promise<void> {
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
switch (newStatus) {
case "pause":
await this.userTriggeredPause();
break;
case "resume": {
const currentStatus = await this.getStatus();
if (currentStatus === InstanceStatus.WaitingForPause) {
// Engine is still running — cancel the pending pause
this.timeoutHandler.cancelWaitingPromisesByType("pause");
await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.Running);
} else if (currentStatus === InstanceStatus.Paused) {
await this.attemptResume();
}
break;
}
case "terminate": {
const currentStatus = await this.getStatus();
if (
[InstanceStatus.Terminated, InstanceStatus.Complete, InstanceStatus.Errored].includes(
currentStatus,
)
) {
throw createWorkflowError(
"Cannot terminate instance since its on a finite state",
"instance.cannot_terminate",
);
}
await this.userTriggeredTerminate(terminateOptions);
break;
}
case "restart":
if (from) {
if (!resolveGroupKeysToWipe(this.ctx.storage.sql, from)) {
throw stepNotFoundError(from.name);
}
await storeRestartFromStep(this.ctx.storage, from);
}
await this.userTriggeredRestart();
break;
}
}
private async replayRollbackRegistry(metadata: InstanceMetadata): Promise<void> {
if (this.rollbackRegistry.size > 0) {
return;
}
const eligible = this.getEligibleRollbackSteps();
if (eligible.length === 0) {
return;
}
this.rollbackEligibleCacheKeys = new Set(eligible);
const stubStep = this.createRollbackContext();
this.setRollbackPhase("replay");
try {
await (this.ctx.props.workflowClass ?? this.env.USER_WORKFLOW).run(
metadata.event,
stubStep as unknown as WorkflowStep,
);
} catch (replayErr) {
// Match the production engine: replay may stop on normal workflow control
// flow; rollback execution uses whatever handlers replay registered.
console.debug("Rollback replay stopped:", replayErr);
} finally {
this.setRollbackPhase(undefined);
this.rollbackEligibleCacheKeys = undefined;
}
}
async userTriggeredTerminate(options?: WorkflowInstanceTerminateOptions) {
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
if (options?.rollback === true) {
this.priorityQueue ??= new TimePriorityQueue(this.ctx, metadata);
await this.replayRollbackRegistry(metadata);
const error = new Error("Instance terminated during rollback");
error.name = "Terminated";
this.setRollbackPhase("rollback");
try {
await executeRollbacks(this, error);
} catch (rollbackErr) {
console.error("Rollback execution failed:", rollbackErr);
} finally {
this.setRollbackPhase(undefined);
}
}
this.writeLog(InstanceEvent.WORKFLOW_TERMINATED, null, null, {
trigger: {
source: InstanceTrigger.API,
},
});
await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.Terminated);
await this.abort(ABORT_REASONS.USER_TERMINATE);
}
/** Deletes all instance state and aborts its current execution. */
async deleteInstance(): Promise<void> {
if ((await this.ctx.storage.get(INSTANCE_METADATA)) === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
await this.ctx.storage.deleteAll();
await this.abort(ABORT_REASONS.USER_DELETE);
}
async userTriggeredPause() {
const status = await this.getStatus();
if (status === InstanceStatus.Paused || status === InstanceStatus.WaitingForPause) {
return;
}
if (status !== InstanceStatus.Running && status !== InstanceStatus.Waiting) {
return;
}
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.WaitingForPause);
void this.timeoutHandler
.waitUntilNothingIsRunning("pause", async () => {
await this.ctx.storage.put(PAUSE_DATETIME, new Date());
await this.setStatus(metadata.accountId, metadata.instance.id, InstanceStatus.Paused);
// Signal the pause controller to interrupt any active
// scheduler.wait (sleep/waitForEvent). The workflow will
// throw a pause error at the next step boundary via
// #checkForPendingPause()
this.pauseController.abort(ABORT_REASONS.USER_PAUSE);
})
.catch(() => {});
}
async userTriggeredRestart() {
// cleanup is done in attemptRestart() on the fresh DO instance
await this.abort(ABORT_REASONS.USER_RESTART);
}
async attemptRestart() {
const restartFromStep = await readAndClearRestartFromStep(this.ctx.storage);
let groupKeysToWipe: Set<string> | null = null;
if (restartFromStep) {
groupKeysToWipe = resolveGroupKeysToWipe(this.ctx.storage.sql, restartFromStep);
if (!groupKeysToWipe) {
throw stepNotFoundError(restartFromStep.name);
}
}
await wipeRestartState(this.ctx.storage, ENGINE_STATUS_KEY, PAUSE_DATETIME, groupKeysToWipe);
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
const { accountId, workflow, version, instance, event } = metadata;
if (!groupKeysToWipe) {
this.writeLog(InstanceEvent.WORKFLOW_QUEUED, null, null, {
params: event.payload,
versionId: version.id,
trigger: {
source: InstanceTrigger.API,
},
});
this.writeLog(InstanceEvent.WORKFLOW_START, null, null, {});
}
void this.init(accountId, workflow, version, instance, event);
}
async attemptResume() {
const metadata = await this.ctx.storage.get<InstanceMetadata>(INSTANCE_METADATA);
if (metadata === undefined) {
throw createWorkflowError("Instance does not exist", "instance.not_found");
}
const status = await this.ctx.storage.get<InstanceStatus>(ENGINE_STATUS_KEY);
if (status !== InstanceStatus.Paused) {
return;
}
// Offset priority queue timers by the pause duration so that
// sleeps/retries resume from where they left off
const pausedDate = await this.ctx.storage.get<Date>(PAUSE_DATETIME);
if (pausedDate !== undefined) {
const offset = Date.now() - new Date(pausedDate).valueOf();
if (this.priorityQueue) {
this.priorityQueue.offsetAll(offset);
} else {
const pq = new TimePriorityQueue(this.ctx, metadata);
pq.offsetAll(offset);
}
}
await this.ctx.storage.delete(PAUSE_DATETIME);
const { accountId, workflow, version, instance, event } = metadata;
await this.ctx.storage.put(ENGINE_STATUS_KEY, InstanceStatus.Queued);
// Reset pause state for the new run. The DO stays alive across
// pause/resume (no this.ctx.abort()), so we need to clear stale
// in-memory state from the previous run to avoid duplicates.
this.pauseController = new AbortController();
this.waiters = new Map();
this.eventMap = new Map();
clearRollbackRegistry(this.rollbackRegistry);
void this.init(accountId, workflow, version, instance, event);
}
async hasInstance(): Promise<boolean> {
// INSTANCE_METADATA is written exactly once, by the first init() for this
// id, so its presence is the durable marker that the instance exists.
return (await this.ctx.storage.get(INSTANCE_METADATA)) !== undefined;
}
async init(
accountId: number,
workflow: DatabaseWorkflow,
version: DatabaseVersion,
instance: DatabaseInstance,
event: WorkflowEvent<unknown>,
) {
if (this.priorityQueue === undefined) {
this.priorityQueue = new TimePriorityQueue(
this.ctx,
// this.env,
{
accountId,
workflow,
version,
instance,
event,
},
);
}
if (this.isRunning) {
return;
}
this.priorityQueue.popPastEntries();
await this.priorityQueue.handleNextAlarm();
// We are not running and are possibly starting a new lifetime
this.accountId = accountId;
this.instanceId = instance.id;
this.workflowName = workflow.name;
const status = await this.getStatus();
if (
[
InstanceStatus.Errored, // TODO (WOR-85): Remove this once upgrade story is done
InstanceStatus.Terminated,
InstanceStatus.Complete,
InstanceStatus.Paused,
].includes(status)
) {
return;
}
// If the DO restarted (e.g. from alarm) while in WaitingForPause state,
// transition to Paused and return early — same as production behaviour.
if (status === InstanceStatus.WaitingForPause) {
await this.ctx.storage.put(PAUSE_DATETIME, new Date());
await this.setStatus(accountId, instance.id, InstanceStatus.Paused);
return;
}
if ((await this.ctx.storage.get(INSTANCE_METADATA)) == undefined) {
const instanceMetadata: InstanceMetadata = {
accountId,
workflow,
version,
instance,
event,
};
await this.ctx.storage.put(INSTANCE_METADATA, instanceMetadata);
// TODO (WOR-78): We currently don't have a queue mechanism
this.writeLog(InstanceEvent.WORKFLOW_QUEUED, null, null, {
params: event.payload,
versionId: version.id,
trigger: {
source: InstanceTrigger.API,
},
});
this.writeLog(InstanceEvent.WORKFLOW_START, null, null, {});
}
// restore eventMap so that waitForEvent across lifetimes works correctly
await this.restoreEventMap();
const stubStep = new Context(this, this.ctx);
const workflowRunningHandler = async () => {
await this.ctx.storage.transaction(async () => {
// manually start the grace period
// startGracePeriod(this, this.timeoutHandler.timeoutMs);
await this.setStatus(accountId, instance.id, InstanceStatus.Running);
});
};
this.isRunning = true;
void workflowRunningHandler();
try {
const target = this.ctx.props.workflowClass ?? this.env.USER_WORKFLOW;
const result = await target.run(event, stubStep as unknown as WorkflowStep);
await this.ctx.storage.put(WORKFLOW_OUTPUT_KEY, {
value: normalizeForStorage(result),
});
this.writeLog(InstanceEvent.WORKFLOW_SUCCESS, null, null, {
result,
});
// NOTE(lduarte): we want to run this in a transaction to guarentee ordering with running setstatus call
// in case that it returns immediately
await this.ctx.storage.transaction(async () => {
await this.setStatus(accountId, instance.id, InstanceStatus.Complete);
});
// Dispose dup'd stubs; otherwise they leak across DO lifetimes.
clearRollbackRegistry(this.rollbackRegistry);
this.isRunning = false;
} catch (err) {
if (isAbortError(err)) {
this.isRunning = false;
return;
}
// Run before the terminal status so events land before WORKFLOW_FAILURE.
try {
await executeRollbacks(this, err instanceof Error ? err : new Error(String(err)));
} catch (rollbackErr) {
console.error("Rollback execution failed:", rollbackErr);
}
let error;
if (err instanceof Error) {
if (err.name === "NonRetryableError" || err.message.startsWith("NonRetryableError")) {
const fatalError = shouldPreserveNonRetryableError()
? new PreservedNonRetryableError(err)
: new WorkflowFatalError(
`The execution of the Workflow instance was terminated, as a step threw an NonRetryableError and it was not handled`,
);
this.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
error: fatalError,
});
await this.setStatus(accountId, instance.id, InstanceStatus.Errored);
await this.abort(ABORT_REASONS.NON_RETRYABLE_ERROR);
this.isRunning = false;
return;
}
error = {
message: err.message,
name: err.name,
};
} else {
error = {
name: "Error",
message: err,
};
}
this.writeLog(InstanceEvent.WORKFLOW_FAILURE, null, null, {
error,
});
// NOTE(lduarte): we want to run this in a transaction to guarentee ordering with running setstatus call
// in case that it throws immediately
await this.ctx.storage.transaction(async () => {
await this.setStatus(accountId, instance.id, InstanceStatus.Errored);
});
this.isRunning = false;
}
return {
id: instance.id,
};
}
}