sailfork / t3code-github / apps/ server/ src/ provider/ DocumentText.ts

// @effect-diagnostics nodeBuiltinImport:off -- the isolated parser process is an external runtime boundary.
import * as NodeChildProcess from "node:child_process";
import * as NodeURL from "node:url";
import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Schema from "effect/Schema";
import * as Semaphore from "effect/Semaphore";
import * as HostProcess from "@t3tools/shared/HostProcess";

const extensions = new Set(["docx", "xlsx", "pptx", "odt", "ods", "odp", "rtf", "epub", "ipynb"]);

class DocumentTextError extends Schema.TaggedError<DocumentTextError>()("DocumentTextError", {
  operation: Schema.Literal("extract"),
  path: Schema.String,
  extension: Schema.String,
  cause: Schema.Defect(),
}) {
  override get message(): string {
    return `Failed to ${this.operation} ${this.extension} document text.`;
  }
}

export const resolveExtractedTextPath = Effect.fn("DocumentText.resolveExtractedTextPath")(
  function* (path: string) {
    const extension = path.split(".").at(-1)?.toLowerCase();
    if (!extension || !extensions.has(extension)) return;
    const fs = yield* FileSystem.FileSystem;
    return yield* fs.exists(path + ".txt").pipe(
      Effect.map((exists) => (exists ? path + ".txt" : undefined)),
      Effect.orElseSucceed(() => undefined),
    );
  },
);

export class DocumentText extends Context.Service<
  DocumentText,
  {
    readonly ensure: (path: string) => Effect.Effect<string | undefined>;
  }
>()("t3/provider/DocumentText") {}

const make = Effect.gen(function* () {
  const fs = yield* FileSystem.FileSystem;
  const executable = yield* HostProcess.ExecutablePath;
  const isExecutable = yield* HostProcess.IsExecutable;
  const environment = yield* HostProcess.Environment;
  const clock = yield* Clock.Clock;
  const lock = yield* Semaphore.make(1);
  const failed = new Map<string, number>();
  // In a bundle import.meta.url is bin.mjs; development and tests use this source module.
  const entry = NodeURL.fileURLToPath(
    new URL(import.meta.url.endsWith(".ts") ? "../bin.ts" : "./bin.mjs", import.meta.url),
  );
  const extract = Effect.fn("DocumentText.extract")(function* (path: string, extension: string) {
    const sidecar = `${path}.txt`;
    if (yield* fs.exists(sidecar)) return sidecar;
    if ((failed.get(path) ?? 0) > clock.currentTimeMillisUnsafe()) return;
    const info = yield* fs.stat(path);
    if (Number(info.size) > 16 * 1024 * 1024) return;
    const content = yield* Effect.tryPromise({
      try: (signal) =>
        new Promise<string>((resolve, reject) => {
          NodeChildProcess.execFile(
            executable,
            [...(isExecutable ? [] : [entry]), "__document-text", path, `.${extension}`],
            {
              signal,
              timeout: 15_000,
              maxBuffer: 4 * 1024 * 1024 + 1024,
              encoding: "utf8",
              windowsHide: true,
              env: {
                ...environment,
                ELECTRON_RUN_AS_NODE: "1",
                NODE_OPTIONS: `${environment.NODE_OPTIONS ?? ""} --max-old-space-size=192`,
              },
            },
            (error, stdout) => (error ? reject(error) : resolve(stdout)),
          );
        }),
      catch: (cause) => new DocumentTextError({ operation: "extract", path, extension, cause }),
    }).pipe(
      Effect.tapError(({ cause }) => {
        if (failed.size >= 1024) failed.delete(failed.keys().next().value!);
        failed.set(
          path,
          cause instanceof Error && "code" in cause && cause.code === 2
            ? Infinity
            : clock.currentTimeMillisUnsafe() + 1_000,
        );
        return Effect.logWarning("Document text extraction failed.", { extension });
      }),
    );
    const pending = `${sidecar}.part`;
    yield* fs
      .writeFileString(pending, content)
      .pipe(
        Effect.andThen(fs.rename(pending, sidecar)),
        Effect.ensuring(fs.remove(pending, { force: true }).pipe(Effect.ignore)),
        Effect.uninterruptible,
      );
    return sidecar;
  }, lock.withPermits(1));
  const ensure = Effect.fn("DocumentText.ensure")(
    function* (path: string) {
      const extension = path.split(".").at(-1)?.toLowerCase();
      if (!extension || !extensions.has(extension)) return;
      const sidecar = `${path}.txt`;
      if (yield* fs.exists(sidecar)) return sidecar;
      return yield* extract(path, extension);
    },
    Effect.orElseSucceed(() => undefined),
  );
  return DocumentText.of({ ensure });
});

export const layer = Layer.effect(DocumentText, make);