sailfork / t3code-github / .repos/ effect/ packages/ platform/ node-shared/ test/ NodeStream.test.ts

import * as NodeStream from "@effect/platform-node-shared/NodeStream"
import { assert, describe, it } from "@effect/vitest"
import { Effect } from "effect"
import * as Array from "effect/Array"
import * as ByteSize from "effect/ByteSize"
import * as Channel from "effect/Channel"
import * as Console from "effect/Console"
import * as Stream from "effect/Stream"
import { Duplex, Readable, Transform } from "node:stream"
import * as Zlib from "node:zlib"

describe("Stream", () => {
  const destroyedOnRead = () =>
    new Readable({
      read() {
        this.destroy()
      }
    })
  const errorCode = (error: unknown) => (error as NodeJS.ErrnoException).code

  for (const [chunkSize, expectedSize] of [[undefined, 4096], [2, 2], [Infinity, 10_000]] as const) {
    it.effect(`fromReadable batches synchronous draining (${chunkSize ?? "default 4096"})`, () =>
      Effect.gen(function*() {
        let reads = 0
        const readable = new Readable({
          objectMode: true,
          highWaterMark: 1,
          read() {
            reads++
            this.push(reads > 10_000 ? null : reads)
          }
        })
        const chunks = yield* NodeStream.fromReadable<number>({
          evaluate: () => readable,
          ...(chunkSize === undefined ? {} : { chunkSize })
        }).pipe(Stream.chunks, Stream.take(1), Stream.runCollect)

        assert.strictEqual(chunks.length, 1)
        assert.strictEqual(chunks[0].length, expectedSize)
        assert.deepEqual(chunks, [Array.range(1, expectedSize)])
        // Node may prefetch one object into its own buffer.
        assert.isAtMost(reads, expectedSize + 1)
      }))
  }

  for (
    const [options, expected] of [
      [{ chunkSize: 2, readSize: 3 }, [["abc", "def"], ["ghi", "jk"]]],
      [{ chunkSize: 2 }, [["abcdefghijk"]]],
      [{ readSize: 3 }, [["abc", "def", "ghi", "jk"]]],
      [{ chunkSize: Infinity, readSize: 3 }, [["abc", "def", "ghi", "jk"]]]
    ] as const
  ) {
    it.effect(`fromReadable separates read size from batching (chunkSize=${"chunkSize" in options ? options.chunkSize : "default"}, readSize=${"readSize" in options ? options.readSize : "default"})`, () =>
      Effect.gen(function*() {
        const readable = new Readable({ read() {} })
        readable.push(Buffer.from("abcdefghijk"))
        readable.push(null)
        const chunks = yield* NodeStream.fromReadable({
          evaluate: () => readable,
          ...options
        }).pipe(Stream.chunks, Stream.runCollect)

        assert.deepEqual(
          chunks.map((batch) => batch.map((bytes) => Buffer.from(bytes).toString())),
          expected.map((batch) => [...batch])
        )
      }))
  }

  it.effect("fromReadable fails on premature close", () =>
    Effect.gen(function*() {
      const code = yield* NodeStream.fromReadable({ evaluate: destroyedOnRead, onError: errorCode }).pipe(
        Stream.runDrain,
        Effect.flip
      )
      assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
    }))

  it.effect("fromReadable fails on an already destroyed readable", () =>
    Effect.gen(function*() {
      const readable = new Readable({ read() {} })
      const closed = new Promise((resolve) => readable.once("close", resolve))
      readable.destroy()
      yield* Effect.promise(() => closed)
      const code = yield* NodeStream.fromReadable({ evaluate: () => readable, onError: errorCode }).pipe(
        Stream.runDrain,
        Effect.flip
      )
      assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
    }))

  it.effect("fromReadable removes its listeners when destroyed without a close event", () =>
    Effect.gen(function*() {
      const readable = new Readable({ emitClose: false, read() {} })
      readable.push("a")
      yield* NodeStream.fromReadable({ evaluate: () => readable }).pipe(Stream.take(1), Stream.runDrain)
      yield* Effect.promise(() => new Promise((resolve) => setImmediate(resolve)))
      assert.strictEqual(readable.listenerCount("error"), 0)
      assert.strictEqual(readable.listenerCount("close"), 0)
    }))

  it.effect("toString fails on premature close", () =>
    Effect.gen(function*() {
      const code = yield* NodeStream.toString(destroyedOnRead, { onError: errorCode }).pipe(Effect.flip)
      assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
    }))

  it.effect("toArrayBuffer fails on premature close", () =>
    Effect.gen(function*() {
      const code = yield* NodeStream.toArrayBuffer(destroyedOnRead, { onError: errorCode }).pipe(Effect.flip)
      assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
    }))

  it.effect("toString keeps the maxBytes failure when destroy fails", () =>
    Effect.gen(function*() {
      const readable = new Readable({
        read() {
          this.push("too large")
        },
        destroy(_error, callback) {
          setImmediate(() => callback(new Error("destroy failed")))
        }
      })
      const closed = new Promise((resolve) => readable.once("close", resolve))
      const error = yield* NodeStream.toString(() => readable, {
        maxBytes: 1,
        onError: (error) => (error as Error).message
      }).pipe(Effect.flip)
      assert.strictEqual(error, "maxBytes exceeded")
      yield* Effect.promise(() => closed)
    }))

  it.effect("handles a late upload error after overflow without leaking listeners", () =>
    Effect.gen(function*() {
      const readable = new Readable({ read() {} })
      yield* Effect.addFinalizer(() => Effect.sync(() => readable.destroy()))
      readable.push(Buffer.from("too large"))
      const error = yield* NodeStream.toString(() => readable, {
        maxBytes: 1,
        destroyOnOverflow: false,
        onError: (error) => error
      }).pipe(Effect.flip)

      assert.instanceOf(error, NodeStream.MaxBytesError)
      assert.isFalse(readable.destroyed)
      readable.push(Buffer.from("remaining upload"))
      const closed = new Promise<void>((resolve) => readable.once("close", resolve))
      readable.destroy(new Error("late upload failure"))
      yield* Effect.promise(() => closed)
      for (const event of ["data", "end", "error", "close", "finish"]) {
        assert.strictEqual(readable.listenerCount(event), 0)
      }
    }))

  it.effect("should read a stream", () =>
    Effect.gen(function*() {
      const stream = NodeStream.fromReadable<"error", string>({
        evaluate: () => Readable.from(["a", "b", "c"]),
        onError: () => "error"
      })
      const items = yield* Stream.runCollect(stream)
      assert.deepEqual(items, ["a", "b", "c"])
    }))

  it.effect("fromDuplex", () =>
    Effect.gen(function*() {
      const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
        Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex({
          evaluate: () =>
            new Transform({
              transform(chunk, _encoding, callback) {
                callback(null, chunk.toString().toUpperCase())
              }
            }),
          onError: () => "error" as const
        })),
        Stream.decodeText(),
        Stream.mkString
      )

      assert.strictEqual(result, "ABC")
    }))

  for (const adapter of ["fromDuplex", "pipeThroughDuplex"] as const) {
    it.effect(`${adapter} bounds byte drains and completes in order`, () =>
      Effect.gen(function*() {
        const input = Buffer.from("abcdefghijk")
        let offset = 0
        let reads = 0
        const options = {
          evaluate: () =>
            new Duplex({
              highWaterMark: 1,
              read() {
                reads++
                if (offset === input.length) {
                  this.push(null)
                } else {
                  const next = Math.min(offset + 3, input.length)
                  this.push(input.subarray(offset, next))
                  offset = next
                }
              },
              write(_chunk, _encoding, callback) {
                callback()
              }
            }),
          chunkSize: 2,
          readSize: 3
        }
        const stream = adapter === "fromDuplex"
          ? Stream.empty.pipe(Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex(options)))
          : Stream.empty.pipe(NodeStream.pipeThroughDuplex(options))
        let batches = 0
        const chunks = yield* stream.pipe(
          Stream.chunks,
          Stream.tap((batch) =>
            Effect.sync(() => {
              assert.isAtMost(batch.length, 2)
              if (batches++ === 0) {
                assert.deepEqual(batch.map((bytes) => Buffer.from(bytes).toString()), ["abc", "def"])
                // Allow one Node-prefetched buffer, but not the entire source.
                assert.isAtMost(reads, 3)
              }
            })
          ),
          Stream.runCollect
        )
        assert.isAbove(batches, 1)
        const bytes = chunks.flat()
        assert.deepEqual(bytes.map((chunk) => chunk.length), [3, 3, 3, 2])
        assert.strictEqual(Buffer.concat(bytes).toString(), "abcdefghijk")
      }))
  }

  it.effect("fromDuplex failure", () =>
    Effect.gen(function*() {
      const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
        Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex({
          evaluate: () =>
            new Transform({
              transform(_chunk, _encoding, callback) {
                callback(new Error())
              }
            }),
          onError: () => "error" as const
        })),
        Stream.runDrain,
        Effect.flip
      )

      assert.strictEqual(result, "error")
    }))

  it.effect("pipeThroughDuplex", () =>
    Effect.gen(function*() {
      const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
        NodeStream.pipeThroughDuplex({
          evaluate: () =>
            new Transform({
              transform(chunk, _encoding, callback) {
                callback(null, chunk.toString().toUpperCase())
              }
            }),
          onError: () => "error" as const
        }),
        Stream.decodeText(),
        Stream.mkString
      )

      assert.strictEqual(result, "ABC")
    }))

  it.effect("pipeThroughDuplex propagates upstream failure", () =>
    Effect.gen(function*() {
      const result = yield* Stream.fail("upstream error").pipe(
        NodeStream.pipeThroughDuplex({
          evaluate: () =>
            new Duplex({
              read() {},
              write(_chunk, _encoding, callback) {
                callback()
              }
            })
        }),
        Stream.runDrain,
        Effect.flip
      )

      assert.strictEqual(result, "upstream error")
    }))

  it.effect("pipeThroughDuplex write error", () =>
    Effect.gen(function*() {
      const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
        NodeStream.pipeThroughDuplex({
          evaluate: () =>
            new Duplex({
              read() {},
              write(_chunk, _encoding, callback) {
                callback(new Error())
              }
            }),
          onError: () => "error" as const
        }),
        Stream.runDrain,
        Effect.flip
      )
      assert.strictEqual(result, "error")
    }))

  it.effect("pipeThroughSimple", () =>
    Effect.gen(function*() {
      const result = yield* Stream.fromArray(["a", Buffer.from("b"), "c"]).pipe(
        NodeStream.pipeThroughSimple(
          () =>
            new Transform({
              transform(chunk, _encoding, callback) {
                callback(null, chunk.toString().toUpperCase())
              }
            })
        ),
        Stream.decodeText(),
        Stream.mkString
      )

      assert.strictEqual(result, "ABC")
    }))

  it.effect("fromDuplex should work with node:zlib", () =>
    Effect.gen(function*() {
      const text = "abcdefg1234567890"
      const encoder = new TextEncoder()
      const input = encoder.encode(text)
      const stream = NodeStream.fromReadable<Uint8Array, "error">({
        evaluate: () => Readable.from([input]),
        onError: () => "error"
      })
      const deflate = NodeStream.fromDuplex({
        evaluate: () => Zlib.createGzip(),
        onError: () => "error" as const
      })
      const inflate = NodeStream.fromDuplex<never, Uint8Array, Uint8Array, "error">({
        evaluate: () => Zlib.createUnzip(),
        onError: () => "error" as const
      })
      const channel = Channel.pipeToOrFail(deflate, inflate)
      const result = yield* stream.pipe(
        Stream.pipeThroughChannelOrFail(channel),
        Stream.decodeText(),
        Stream.mkString
      )
      assert.strictEqual(result, text)
    }))

  it.effect("toReadable roundtrip", () =>
    Effect.gen(function*() {
      const stream = Stream.range(0, 10000).pipe(
        Stream.map((n) => String(n))
      )
      const readable = yield* NodeStream.toReadable(stream)
      const outStream = NodeStream.fromReadable({
        evaluate: () => readable,
        onError: () => "error" as const
      })
      const items = yield* outStream.pipe(
        Stream.decodeText(),
        Stream.runCollect
      )
      assert.strictEqual(items.join(""), Array.range(0, 10000).join(""))
    }))

  it.effect("toReadable with error", () =>
    Effect.gen(function*() {
      const stream = Stream.fail("error")
      const readable = yield* NodeStream.toReadable(stream)
      const outStream = NodeStream.fromReadable({
        evaluate: () => readable
      })
      const error = yield* outStream.pipe(
        Stream.runCollect,
        Effect.tapError((_) => Console.log(_)),
        Effect.flip
      )
      assert.deepEqual(error.cause, "error")
    }))

  it.effect("toString registers one error listener", () =>
    Effect.gen(function*() {
      const stream = new Readable({
        read() {}
      })
      yield* NodeStream.toString(() => stream).pipe(Effect.forkChild)
      yield* Effect.yieldNow
      assert.strictEqual(stream.listenerCount("error"), 1)
    }))

  it.effect("collects strings and array buffers with an Infinity maxBytes limit", () =>
    Effect.gen(function*() {
      const chunks = [Buffer.from("hello "), Buffer.from("world")]
      const text = yield* NodeStream.toString(() => Readable.from(chunks), { maxBytes: Infinity })
      const buffer = yield* NodeStream.toArrayBuffer(() => Readable.from(chunks), { maxBytes: Infinity })

      assert.strictEqual(text, "hello world")
      assert.deepStrictEqual(new Uint8Array(buffer), new TextEncoder().encode("hello world"))
    }))

  it.effect("toString enforces a zero maxBytes limit", () =>
    Effect.gen(function*() {
      let emitted = false
      const stream = new Readable({
        read() {
          if (emitted) return
          emitted = true
          this.push("a")
        }
      })
      const error = yield* NodeStream.toString(() => stream, {
        maxBytes: "0 B",
        onError: () => "maxBytes exceeded" as const
      }).pipe(Effect.flip)

      assert.strictEqual(error, "maxBytes exceeded")
      assert.isTrue(stream.destroyed)
    }))

  it.effect("toString limits the bytes read before decoding", () =>
    Effect.gen(function*() {
      const latin1 = yield* NodeStream.toString(() => Readable.from([Buffer.from([0xe9])]), {
        encoding: "latin1",
        maxBytes: 1
      })
      const utf16le = yield* NodeStream.toString(() => Readable.from([Buffer.from("abcd", "utf16le")]), {
        encoding: "utf16le",
        maxBytes: 7,
        onError: (error) => error
      }).pipe(Effect.flip)

      assert.strictEqual(latin1, "é")
      assert.instanceOf(utf16le, NodeStream.MaxBytesError)
    }))

  it.effect("toString decodes characters split across chunks", () =>
    Effect.gen(function*() {
      const chunks = [Buffer.from([0xe2, 0x82]), Buffer.from([0xac, 0xe2])]
      const text = yield* NodeStream.toString(() => Readable.from(chunks), { maxBytes: 4 })

      assert.strictEqual(text, "€\ufffd")
    }))

  it.effect("toString limits the bytes read by a stream that already decodes", () =>
    Effect.gen(function*() {
      const stream = new Readable({
        read() {
          this.push(Buffer.from([0xe9]))
          this.push(null)
        }
      })
      stream.setEncoding("utf8")
      const text = yield* NodeStream.toString(() => stream, { encoding: "latin1", maxBytes: 1 })

      assert.strictEqual(text, "é")
    }))

  it.effect("toString limits object-mode strings by their UTF-8 length", () =>
    Effect.gen(function*() {
      const error = yield* NodeStream.toString(() => Readable.from(["a", "b", "c", "d"]), {
        encoding: "hex",
        maxBytes: 3,
        onError: (error) => error
      }).pipe(Effect.flip)

      assert.instanceOf(error, NodeStream.MaxBytesError)
    }))

  it.effect("toArrayBuffer enforces a zero maxBytes limit", () =>
    Effect.gen(function*() {
      let emitted = false
      const stream = new Readable({
        read() {
          if (emitted) return
          emitted = true
          this.push(Buffer.from("a"))
        }
      })
      const error = yield* NodeStream.toArrayBuffer(() => stream, {
        maxBytes: ByteSize.zero,
        onError: () => "maxBytes exceeded" as const
      }).pipe(Effect.flip)

      assert.strictEqual(error, "maxBytes exceeded")
      assert.isTrue(stream.destroyed)
    }))
})