sailfork / t3code-github / apps/ server/ src/ orchestration-v2/ LiveStreamBudget.ts

import * as Schema from "effect/Schema";
import * as Arr from "effect/Array";
import type * as Cause from "effect/Cause";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as Queue from "effect/Queue";
import * as PubSub from "effect/PubSub";
import * as Scope from "effect/Scope";
import * as Semaphore from "effect/Semaphore";
import * as Stream from "effect/Stream";

export class LiveStreamBufferError extends Schema.TaggedError<LiveStreamBufferError>()(
  "LiveStreamBufferError",
  { message: Schema.String },
) {}

export const LIVE_STREAM_MAX_ITEMS = 1_000;
const LIVE_STREAM_MAX_SERIALIZED_BYTES = 8 * 1024 * 1024;

export interface RetainedLiveItem<A> {
  readonly value: A;
  readonly serializedBytes: number;
}

// Immutable events are shared across subscribers; cache sizes without retaining events or serialized copies.
const serializedSizes = new WeakMap<object, number>();

function serializedSize(value: object): number {
  const cached = serializedSizes.get(value);
  if (cached !== undefined) {
    return cached;
  }
  const bytes = Buffer.byteLength(JSON.stringify(value));
  serializedSizes.set(value, bytes);
  return bytes;
}

export const makeLiveStreamBudget = Effect.fn("makeLiveStreamBudget")(function* (limits?: {
  readonly maxItems?: number;
  readonly maxSerializedBytes?: number;
}) {
  const maxItems = limits?.maxItems ?? LIVE_STREAM_MAX_ITEMS;
  const maxSerializedBytes = limits?.maxSerializedBytes ?? LIVE_STREAM_MAX_SERIALIZED_BYTES;
  const failed = yield* Deferred.make<never, LiveStreamBufferError>();
  const cleanupComplete = yield* Deferred.make<void>();
  let failure: LiveStreamBufferError | undefined;
  const retained = new Set<RetainedLiveItem<unknown>>();
  let retainedSerializedBytes = 0;

  const release = (items: Iterable<RetainedLiveItem<unknown>>) => {
    for (const item of items) {
      if (!retained.delete(item)) {
        continue;
      }
      retainedSerializedBytes -= item.serializedBytes;
    }
  };
  yield* Effect.addFinalizer(() =>
    Effect.sync(() => release(retained)).pipe(
      Effect.andThen(Deferred.succeed(cleanupComplete, undefined)),
    ),
  );

  const check = Effect.suspend(() => (failure ? Effect.fail(failure) : Effect.void));

  const overflow = Effect.fn("LiveStreamBudget.overflow")(function* (
    nextItems: number,
    nextSerializedBytes: number,
  ) {
    failure ??= new LiveStreamBufferError({
      message: "The live event buffer is full. Resume from the last received sequence.",
    });
    yield* Deferred.fail(failed, failure);
    yield* Effect.logWarning("orchestration live event buffer is full", {
      retainedItems: retained.size,
      retainedSerializedBytes,
      nextItems,
      nextSerializedBytes,
      maxItems,
      maxSerializedBytes,
    });
    return yield* failure;
  });

  const retain = <A extends object>(value: A, payload: object = value) =>
    Effect.suspend(() => {
      if (failure) {
        return Effect.fail(failure);
      }
      const serializedBytes = serializedSize(payload);
      const nextItems = retained.size + 1;
      const nextSerializedBytes = retainedSerializedBytes + serializedBytes;
      if (nextItems > maxItems || nextSerializedBytes > maxSerializedBytes) {
        return overflow(nextItems, nextSerializedBytes);
      }
      const item = { value, serializedBytes };
      retained.add(item);
      retainedSerializedBytes = nextSerializedBytes;
      return Effect.succeed(item);
    });

  const replace = <A extends object>(
    previous: ReadonlyArray<RetainedLiveItem<unknown>>,
    values: ReadonlyArray<A>,
    payload: (value: A) => object = (value) => value,
  ) =>
    Effect.suspend(() => {
      if (failure) {
        return Effect.fail(failure);
      }
      const next = values.map((value) => ({
        value,
        serializedBytes: serializedSize(payload(value)),
      }));
      let nextItems = retained.size + next.length;
      let nextSerializedBytes =
        retainedSerializedBytes + next.reduce((sum, item) => sum + item.serializedBytes, 0);
      for (const item of previous) {
        if (retained.has(item)) {
          nextItems -= 1;
          nextSerializedBytes -= item.serializedBytes;
        }
      }
      if (nextItems > maxItems || nextSerializedBytes > maxSerializedBytes) {
        return overflow(nextItems, nextSerializedBytes);
      }
      release(previous);
      for (const item of next) {
        retained.add(item);
      }
      retainedSerializedBytes = nextSerializedBytes;
      return Effect.succeed(next);
    });

  const deliver = <A, E, R>(stream: Stream.Stream<RetainedLiveItem<A>, E, R>) =>
    Stream.fromPull(
      Effect.gen(function* () {
        yield* check;
        const sourceScope = yield* Scope.fork(yield* Effect.scope);
        const source = {
          pull: yield* Stream.toPull(stream).pipe(Scope.provide(sourceScope)),
        };
        let inFlight: ReadonlyArray<RetainedLiveItem<A>> = [];
        yield* Effect.addFinalizer(() =>
          Effect.sync(() => {
            release(inFlight);
            inFlight = [];
            source.pull = Effect.interrupt;
          }),
        );
        yield* Deferred.await(failed).pipe(
          Effect.catchTags({
            LiveStreamBufferError: (error) =>
              Effect.sync(() => {
                // Release retained source chunks without waiting for an RPC ACK.
                source.pull = Effect.interrupt;
              }).pipe(
                Effect.andThen(Scope.close(sourceScope, Exit.fail(error))),
                Effect.andThen(
                  Effect.sync(() => {
                    const delivered = new Set<RetainedLiveItem<unknown>>(inFlight);
                    for (const item of retained) {
                      if (!delivered.has(item)) {
                        release([item]);
                      }
                    }
                  }),
                ),
                Effect.andThen(Deferred.succeed(cleanupComplete, undefined)),
              ),
          }),
          Effect.forkScoped,
        );
        return Effect.gen(function* () {
          // RpcServer requests the next batch after ACK; dequeueing does not end delivery.
          release(inFlight);
          inFlight = [];
          yield* check;
          const items = yield* Effect.raceFirst(source.pull, Deferred.await(failed));
          inFlight = items;
          yield* check;
          return Arr.map(items, (item) => item.value);
        });
      }),
    ).pipe(Stream.scoped);

  return {
    retain,
    replace,
    release,
    deliver,
    check,
    failed: Deferred.await(failed),
    closed: Deferred.await(cleanupComplete),
    usage: Effect.sync(() => ({ retainedItems: retained.size, retainedSerializedBytes })),
  };
});

type LiveStreamBudget = Effect.Success<ReturnType<typeof makeLiveStreamBudget>>;

type LiveStreamLimits = {
  readonly maxItems?: number;
  readonly maxSerializedBytes?: number;
};

const retainLiveStream = Effect.fn("LiveStreamBudget.retainLiveStream")(function* <
  A extends object,
  E,
  R,
>(source: Stream.Stream<A, E, R>, budget: LiveStreamBudget) {
  const queue = yield* Queue.unbounded<
    RetainedLiveItem<A>,
    E | LiveStreamBufferError | Cause.Done
  >();
  let closed = false;
  const close = Effect.fn("LiveStreamBudget.close")(function* (error?: LiveStreamBufferError) {
    if (closed) return;
    closed = true;
    budget.release(yield* Queue.clear(queue).pipe(Effect.orElseSucceed(() => [])));
    if (error) yield* Queue.fail(queue, error);
    yield* Queue.shutdown(queue);
  });
  yield* Effect.addFinalizer(() => close());
  yield* budget.failed.pipe(Effect.catchTags({ LiveStreamBufferError: close }), Effect.forkScoped);
  yield* source.pipe(
    Stream.runForEach((value) =>
      budget.retain(value).pipe(
        Effect.flatMap((item) => Queue.offer(queue, item)),
        Effect.uninterruptible,
      ),
    ),
    Effect.raceFirst(budget.failed),
    Effect.exit,
    Effect.flatMap((exit) =>
      Exit.isFailure(exit) ? Queue.failCause(queue, exit.cause) : Queue.end(queue),
    ),
    Effect.forkScoped({ startImmediately: true }),
  );
  return Stream.fromQueue(queue);
});

/** Coalesce queued full states; delivered states stay charged until the next pull ACKs them. */
export const bufferLatestLiveStream = <A extends { readonly sequence: number }, E, R>(
  source: Stream.Stream<A, E, R>,
  key: (value: A) => string,
  limits?: LiveStreamLimits,
) =>
  Stream.unwrap(
    Effect.gen(function* () {
      const budget = yield* makeLiveStreamBudget(limits);
      const ready = yield* Queue.bounded<void, E | LiveStreamBufferError | Cause.Done>(1);
      const mutex = yield* Semaphore.make(1);
      const pending = new Map<string, RetainedLiveItem<A>>();
      let closed = false;
      const close = (error?: LiveStreamBufferError) =>
        mutex.withPermit(
          Effect.gen(function* () {
            if (closed) return;
            closed = true;
            budget.release(pending.values());
            pending.clear();
            if (error) yield* Queue.fail(ready, error);
            yield* Queue.shutdown(ready);
          }),
        );
      yield* Effect.addFinalizer(() => close());
      yield* budget.failed.pipe(
        Effect.catchTags({ LiveStreamBufferError: close }),
        Effect.forkScoped,
      );
      yield* source.pipe(
        Stream.runForEach((value) =>
          mutex.withPermit(
            Effect.gen(function* () {
              yield* budget.check;
              if (closed) return;
              const identity = key(value);
              const previous = pending.get(identity);
              if (previous && previous.value.sequence >= value.sequence) return;
              const wasEmpty = pending.size === 0;
              const [item] = yield* budget.replace(previous ? [previous] : [], [value]);
              pending.set(identity, item!);
              // Only the empty-to-nonempty transition needs a wakeup; replacements
              // must not leave another token after this batch has drained.
              if (wasEmpty) yield* Queue.offer(ready, undefined);
            }).pipe(Effect.uninterruptible),
          ),
        ),
        Effect.raceFirst(budget.failed),
        Effect.exit,
        Effect.flatMap((exit) =>
          Exit.isFailure(exit) ? Queue.failCause(ready, exit.cause) : Queue.end(ready),
        ),
        Effect.forkScoped({ startImmediately: true }),
      );
      return budget.deliver(
        Stream.fromPull(
          Effect.succeed(
            Queue.take(ready).pipe(
              Effect.andThen(
                mutex.withPermit(
                  Effect.sync(() => {
                    const items = Array.from(pending.values()).sort(
                      (left, right) => left.value.sequence - right.value.sequence,
                    );
                    pending.clear();
                    // Moving to delivery preserves the budget charge until ACK.
                    return items as Arr.NonEmptyArray<RetainedLiveItem<A>>;
                  }),
                ),
              ),
            ),
          ),
        ),
      );
    }),
  );

/**
 * Subscribe and start draining before reading the high-water mark. A blocked
 * catch-up query or an unacknowledged replay batch must not strand an unbounded
 * PubSub subscription.
 */
type ReplayLiveInput<A extends { readonly sequence: number }, E, R> = {
  readonly subscribe: Effect.Effect<PubSub.Subscription<A>, never, Scope.Scope>;
  readonly latestSequence: Effect.Effect<number, E, R>;
  readonly replay: (throughSequence: number) => Stream.Stream<A, E, R>;
  readonly afterSequence?: number;
  readonly filter?: (event: A) => boolean;
};

export const replayAndBufferProjectedLiveEvents = <
  A extends { readonly sequence: number },
  B extends { readonly sequence: number },
  E,
  R,
>(
  input: ReplayLiveInput<A, E, R> & { readonly project: (event: A) => B },
  limits?: LiveStreamLimits,
) =>
  Stream.unwrap(
    Effect.gen(function* () {
      const budget = yield* makeLiveStreamBudget(limits);
      const subscriptionScope = yield* Scope.fork(yield* Effect.scope);
      const subscription = yield* input.subscribe.pipe(Scope.provide(subscriptionScope));
      let throughSequence = input.afterSequence ?? 0;
      const live = yield* retainLiveStream(
        // Take one event at a time so no uncharged PubSub batch is retained.
        Stream.fromEffectRepeat(PubSub.take(subscription)).pipe(
          Stream.filter(
            (event) => event.sequence > throughSequence && (input.filter?.(event) ?? true),
          ),
          Stream.map(input.project),
          Stream.ensuring(Scope.close(subscriptionScope, Exit.void)),
        ),
        budget,
      );
      const replay = Stream.unwrap(
        input.latestSequence.pipe(
          Effect.map((highWater) => {
            throughSequence = Math.max(highWater, input.afterSequence ?? 0);
            return input.replay(highWater).pipe(
              // The finite database page is independent of the live tail.
              // Charge only the item being delivered so a fast reader can
              // consume a page larger than the budget without overflowing.
              Stream.filter((event) => input.filter?.(event) ?? true),
              Stream.rechunk(1),
              Stream.mapEffect((event) => budget.retain(input.project(event))),
            );
          }),
        ),
      );
      return budget.deliver(
        Stream.concat(
          replay,
          live.pipe(
            Stream.filter((item) => {
              if (item.value.sequence > throughSequence) return true;
              budget.release([item]);
              return false;
            }),
          ),
        ),
      );
    }),
  );