sailfork / t3code-github / .repos/ effect/ packages/ platform/ node/ test/ cluster/ ShardFence.integration.test.ts

import { NodeCrypto } from "@effect/platform-node"
import { assert, describe, it } from "@effect/vitest"
import { Clock, Context, Deferred, Effect, Latch, Layer, Option, Schema } from "effect"
import {
  ClusterSchema,
  Entity,
  EntityId,
  MessageStorage,
  RunnerAddress,
  RunnerHealth,
  Runners,
  Sharding,
  ShardingConfig,
  type Snowflake,
  SqlMessageStorage,
  SqlRunnerStorage
} from "effect/cluster"
import { Rpc } from "effect/rpc"
import { SqlClient } from "effect/sql"
import { TestClock } from "effect/testing"
import { PgContainer } from "../fixtures/pg-utils.ts"

const Hold = Entity.make("Hold", [
  Rpc.make("Hold", { success: Schema.String })
]).annotateRpcs(ClusterSchema.Persisted, true)

interface RunnerProbe {
  readonly port: number
  readonly entered: Deferred.Deferred<Snowflake.Snowflake>
  readonly gate: Latch.Latch
  readonly returned: Deferred.Deferred<void>
}

const makeProbe = (port: number): RunnerProbe => ({
  port,
  entered: Deferred.makeUnsafe(),
  gate: Latch.makeUnsafe(false),
  returned: Deferred.makeUnsafe()
})

const HoldLayer = (probe: RunnerProbe) =>
  Hold.toLayer(Effect.gen(function*() {
    const address = yield* Entity.CurrentRunnerAddress
    return Hold.of({
      Hold: Effect.fnUntraced(function*(envelope) {
        yield* Deferred.succeed(probe.entered, envelope.requestId)
        yield* probe.gate.await
        yield* Deferred.succeed(probe.returned, void 0)
        return String(address.port)
      })
    })
  }))

// A clock whose timers stop firing while frozen, as in a paused process: the
// runner's lock refreshes, heartbeats and storage polls stall, while work that
// does not wait on a timer, such as a running handler, carries on.
const makeFreezableClock = () => {
  const live = Clock.Clock.defaultValue()
  const thawed = Latch.makeUnsafe(true)
  const clock: Clock.Clock = {
    currentTimeMillisUnsafe: () => live.currentTimeMillisUnsafe(),
    currentTimeMillis: live.currentTimeMillis,
    currentTimeNanosUnsafe: () => live.currentTimeNanosUnsafe(),
    currentTimeNanos: live.currentTimeNanos,
    monotonicTimeNanosUnsafe: () => live.monotonicTimeNanosUnsafe(),
    monotonicTimeNanos: live.monotonicTimeNanos,
    sleep: (duration) => Effect.andThen(live.sleep(duration), thawed.await)
  }
  return {
    clock,
    freeze: Effect.sync(() => {
      thawed.closeUnsafe()
    }),
    thaw: thawed.open
  }
}

const RunnerLayer = (probe: RunnerProbe, config: Partial<ShardingConfig.ShardingConfig["Service"]>) =>
  HoldLayer(probe).pipe(
    Layer.provideMerge(Sharding.layer),
    Layer.provide([SqlRunnerStorage.layer, RunnerHealth.layerNoop, Runners.layerNoop]),
    Layer.provideMerge(SqlMessageStorage.layer),
    Layer.provide(NodeCrypto.layer),
    Layer.provide(ShardingConfig.layer({
      ...config,
      runnerAddress: Option.some(RunnerAddress.make("localhost", probe.port)),
      shardsPerGroup: 1,
      // Short enough for B to take over quickly, with lock operation deadlines
      // that tolerate CI latency.
      shardLockExpiration: 3000,
      shardLockRefreshInterval: 1000,
      entityMessagePollInterval: 100,
      refreshAssignmentsInterval: 100,
      entityTerminationTimeout: 0
    }))
  )

const waitFor = Effect.fnUntraced(function*<A, E, R>(effect: Effect.Effect<A | undefined, E, R>) {
  while (true) {
    const value = yield* effect
    if (value !== undefined) return value
    yield* Effect.sleep(20)
  }
})

const staleOwnerTest = Effect.fnUntraced(
  function*(options: { readonly advisory: boolean }) {
    const sql = yield* SqlClient.SqlClient
    const config = { shardLockDisableAdvisory: !options.advisory }
    const pauseA = makeFreezableClock()
    const probeA = makeProbe(1234)
    const probeB = makeProbe(5678)

    const contextA = yield* Layer.build(RunnerLayer(probeA, config)).pipe(
      Effect.provideService(Clock.Clock, pauseA.clock)
    )
    const storage = Context.get(contextA, MessageStorage.MessageStorage)
    yield* Hold.client.pipe(
      Effect.flatMap((makeClient) => makeClient("1").Hold(void 0, { discard: true })),
      Effect.provide(contextA)
    )
    const requestId = yield* Deferred.await(probeA.entered).pipe(
      Effect.timeoutOrElse({ duration: 10_000, orElse: () => Effect.die("runner A never started the request") })
    )

    // A is paused, and its lock session is lost.
    yield* pauseA.freeze
    if (options.advisory) {
      yield* sql`SELECT pg_terminate_backend(pid, 10000) FROM pg_locks
        WHERE locktype = 'advisory' AND objsubid = 2
        AND database = (SELECT oid FROM pg_database WHERE datname = current_database())`
    }

    const contextB = yield* Layer.build(RunnerLayer(probeB, config))
    const shardingB = Context.get(contextB, Sharding.Sharding)
    const shardId = shardingB.getShardId(EntityId.make("1"), "default")
    yield* waitFor(Effect.sync(() => shardingB.hasShardId(shardId) || undefined)).pipe(
      Effect.timeoutOrElse({ duration: 15_000, orElse: () => Effect.die("runner B never acquired the shard") })
    )
    yield* Deferred.await(probeB.entered).pipe(
      Effect.timeoutOrElse({ duration: 15_000, orElse: () => Effect.die("runner B never started the request") })
    )

    // A's handler finishes before A's lock refresh has run again.
    yield* probeA.gate.open
    yield* Deferred.await(probeA.returned)

    const storedExit = storage.repliesForUnfiltered([requestId]).pipe(
      Effect.map((replies) => replies.find((reply) => reply._tag === "WithExit"))
    )
    // An unfenced write lands within milliseconds of the handler returning.
    const stale = yield* waitFor(storedExit).pipe(Effect.timeoutOption(2000))
    assert.isTrue(Option.isNone(stale), "the stale owner stored its reply")

    yield* probeB.gate.open
    const reply = yield* waitFor(storedExit).pipe(
      Effect.timeoutOrElse({ duration: 10_000, orElse: () => Effect.die("runner B never stored its reply") })
    )
    assert(reply._tag === "WithExit")
    assert.deepStrictEqual(reply.exit, { _tag: "Success", value: "5678" })

    // Resume A so it can shut down. It may legitimately own the shard again
    // from here, so nothing about it is asserted.
    yield* pauseA.thaw
  },
  // Release both runners before the database.
  Effect.scoped,
  Effect.provide(PgContainer.layerClientDatabase),
  TestClock.withLive
)

describe("Sharding shard fencing", { concurrent: false }, () => {
  // Each test runs two runners against live lock deadlines, so keep them apart.
  it.layer(PgContainer.layer, { concurrent: false, timeout: 120_000 })((it) => {
    it.effect(
      "a runner that lost its advisory lock cannot store a reply",
      () => staleOwnerTest({ advisory: true }),
      60_000
    )

    it.effect("a runner that lost its lease cannot store a reply", () => staleOwnerTest({ advisory: false }), 60_000)
  })
})