sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ BigQuery/ TableHttp.ts

import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
import * as Effect from "effect/Effect";
import * as GCPBindings from "../GCPBindings.ts";
import { type BindingIam, grantFor } from "../HttpBinding.ts";
import { type ListTableResult, QueryNotComplete, type ReadTableClient } from "./ReadTable.ts";
import { decodeRows, encodeRow, queryParameter } from "./Rows.ts";
import type { Table } from "./Table.ts";
import { InsertRowsFailed, type WriteTableClient } from "./WriteTable.ts";

// bigquery.jobs.create is only grantable on the project; data access is
// granted on the table's own IAM policy.
export const readTableIam: ReadonlyArray<BindingIam> = [
  { role: "roles/bigquery.dataViewer", on: "bigquery.table" },
  { role: "roles/bigquery.jobUser" },
];
export const writeTableIam: ReadonlyArray<BindingIam> = [
  { role: "roles/bigquery.dataEditor", on: "bigquery.table" },
];
export const readWriteTableIam: ReadonlyArray<BindingIam> = [
  { role: "roles/bigquery.dataEditor", on: "bigquery.table" },
  { role: "roles/bigquery.jobUser" },
];

export interface TableRef {
  project: string;
  datasetId: string;
  tableId: string;
  location: string | undefined;
}

// `getQueryResults` long-polls up to `timeoutMs` per call, so this bounds
// the total wait for a query job at ~2 minutes.
const QUERY_WAIT_MS = 10_000;
const QUERY_POLLS = 12;

interface QueryPage {
  jobComplete?: boolean;
  schema?: bigquery.TableSchema;
  rows?: bigquery.TableRow[];
  pageToken?: string;
}

/** Fetch every result page after `pageToken`. */
const remainingRows = <E>(
  fetchPage: (pageToken: string) => Effect.Effect<QueryPage, E>,
  pageToken: string | undefined,
): Effect.Effect<ReadonlyArray<bigquery.TableRow>, E> =>
  pageToken
    ? fetchPage(pageToken).pipe(
        Effect.flatMap((page) =>
          remainingRows(fetchPage, page.pageToken).pipe(
            Effect.map((rest) => [...(page.rows ?? []), ...rest]),
          ),
        ),
      )
    : Effect.succeed([]);

/**
 * Shared HTTP scaffolding for the BigQuery Read/Write/ReadWrite table
 * bindings: resolves the distilled operations once at Layer construction
 * and grants the level's roles at deploy time.
 *
 * NOT exported from `index.ts`.
 */
export const makeBigQueryTableHelpers = Effect.gen(function* () {
  const getTable = yield* GCPBindings.withCredentials(bigquery.getTables);
  const listTabledata = yield* GCPBindings.withCredentials(bigquery.listTabledata);
  const queryJobs = yield* GCPBindings.withCredentials(bigquery.queryJobs);
  const getQueryResults = yield* GCPBindings.withCredentials(bigquery.getQueryResultsJobs);
  const insertAll = yield* GCPBindings.withCredentials(bigquery.insertAllTabledata);

  const makeRead = (table: Effect.Effect<TableRef>): ReadTableClient => ({
    list: (
      options,
    ): Effect.Effect<ListTableResult, bigquery.GetTablesError | bigquery.ListTabledataError> =>
      Effect.gen(function* () {
        const ref = yield* table;
        const key = {
          projectId: ref.project,
          datasetId: ref.datasetId,
          tableId: ref.tableId,
        };
        const metadata = yield* getTable({ ...key, view: "BASIC" });
        const page = yield* listTabledata({
          ...key,
          maxResults: options?.maxResults,
          pageToken: options?.pageToken,
          "formatOptions.useInt64Timestamp": true,
        });
        return {
          rows: decodeRows(metadata.schema, page.rows),
          nextPageToken: page.pageToken || undefined,
        };
      }),
    query: (
      sql,
      params,
    ): Effect.Effect<
      Record<string, unknown>[],
      bigquery.QueryJobsError | bigquery.GetQueryResultsJobsError | QueryNotComplete
    > =>
      Effect.gen(function* () {
        const ref = yield* table;
        const first = yield* queryJobs({
          projectId: ref.project,
          body: {
            query: sql,
            useLegacySql: false,
            location: ref.location,
            defaultDataset: {
              projectId: ref.project,
              datasetId: ref.datasetId,
            },
            timeoutMs: QUERY_WAIT_MS,
            formatOptions: { useInt64Timestamp: true },
            ...(params === undefined
              ? {}
              : {
                  parameterMode: "NAMED",
                  queryParameters: Object.entries(params).map(([name, value]) =>
                    queryParameter(name, value),
                  ),
                }),
          },
        });
        const jobId = first.jobReference?.jobId;
        const location = first.jobReference?.location ?? ref.location;
        const results = (
          pageToken: string | undefined,
        ): Effect.Effect<
          bigquery.GetQueryResultsResponse,
          bigquery.GetQueryResultsJobsError | QueryNotComplete
        > =>
          jobId === undefined
            ? Effect.fail(new QueryNotComplete({ jobId }))
            : getQueryResults({
                projectId: ref.project,
                jobId,
                location,
                pageToken,
                timeoutMs: QUERY_WAIT_MS,
                "formatOptions.useInt64Timestamp": true,
              });

        const settled: Effect.Effect<
          QueryPage,
          bigquery.GetQueryResultsJobsError | QueryNotComplete
        > =
          first.jobComplete === true
            ? Effect.succeed(first)
            : results(undefined).pipe(
                Effect.repeat({
                  until: (r: bigquery.GetQueryResultsResponse) => r.jobComplete === true,
                  times: QUERY_POLLS,
                }),
              );
        const complete = yield* settled;
        if (complete.jobComplete !== true) {
          return yield* new QueryNotComplete({ jobId });
        }
        const schema = complete.schema;
        const rest: ReadonlyArray<bigquery.TableRow> = yield* remainingRows(
          results,
          complete.pageToken,
        );
        return decodeRows(schema, [...(complete.rows ?? []), ...rest]);
      }),
  });

  const makeWrite = (table: Effect.Effect<TableRef>): WriteTableClient => ({
    insert: (rows, options) =>
      Effect.gen(function* () {
        if (rows.length === 0) return;
        const ref = yield* table;
        const response = yield* insertAll({
          projectId: ref.project,
          datasetId: ref.datasetId,
          tableId: ref.tableId,
          body: {
            rows: rows.map((row, index) => ({
              json: encodeRow(row),
              insertId: options?.insertIds?.[index],
            })),
          },
        });
        const insertErrors = response.insertErrors ?? [];
        if (insertErrors.length > 0) {
          return yield* new InsertRowsFailed({
            insertErrors: insertErrors.map((item) => ({
              index: item.index,
              errors: item.errors ?? [],
            })),
          });
        }
      }),
  });

  return { makeRead, makeWrite };
});

/** Build a table binding that grants `iam` and returns `makeClient`'s client. */
export const makeBigQueryTableBinding = <Client>(options: {
  tag: string;
  iam: ReadonlyArray<BindingIam>;
  makeClient: (
    helpers: Effect.Success<typeof makeBigQueryTableHelpers>,
    table: Effect.Effect<TableRef>,
  ) => Client;
}) =>
  Effect.gen(function* () {
    const bind = yield* GCPBindings.Bind;
    const helpers = yield* makeBigQueryTableHelpers;
    return Effect.fn(function* (table: Table) {
      if (!globalThis.__ALCHEMY_RUNTIME__) {
        yield* bind`${options.tag}(${table})`({
          iam: options.iam.map((iam) => grantFor(iam, table.name)),
        });
      }
      const project = yield* table.project;
      const datasetId = yield* table.datasetId;
      const tableId = yield* table.tableId;
      const location = yield* table.location;
      const ref = Effect.all({
        project,
        datasetId,
        tableId,
        location,
      });
      return options.makeClient(helpers, ref);
    });
  });