sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Basin/ internal/ schema.ts

import type * as Iceberg from "@distilled.cloud/iceberg";
import * as Data from "effect/Data";
import * as S from "effect/Schema";
import * as AST from "effect/SchemaAST";
import type {
  TableColumn,
  TableColumnType,
  TablePrimitiveType,
  TableSchema,
  TableSchemaInput,
} from "../Table.ts";

/**
 * An Effect Schema (or column) that has no Iceberg representation.
 */
export class BasinTableSchemaUnsupported extends Data.TaggedError(
  "Cloudflare.Basin.TableSchemaUnsupported",
)<{
  message: string;
  path: string;
}> {}

const unsupported = (path: string, what: string) =>
  new BasinTableSchemaUnsupported({
    path,
    message: `Cloudflare.Basin.Table: column '${path || "<root>"}' is ${what}, which has no Iceberg type`,
  });

// ---------------------------------------------------------------------------
// Effect Schema → column descriptor
// ---------------------------------------------------------------------------

const INT32_MIN = -2147483648;
const INT32_MAX = 2147483647;

interface CheckRep {
  id: string;
  payload: unknown;
}

const checkReps = (ast: AST.AST): CheckRep[] => {
  const out: CheckRep[] = [];
  const visit = (check: unknown) => {
    const c = check as {
      annotations?: { representation?: CheckRep };
      checks?: readonly unknown[];
    };
    if (c.annotations?.representation?.id) out.push(c.annotations.representation);
    c.checks?.forEach(visit);
  };
  ((ast as { checks?: readonly unknown[] }).checks ?? []).forEach(visit);
  return out;
};

const hasCheck = (reps: CheckRep[], id: string) => reps.some((r) => r.id === `effect/schema/${id}`);

const numberType = (ast: AST.AST): TablePrimitiveType => {
  const reps = checkReps(ast);
  if (!hasCheck(reps, "isInt")) return "double";
  const fitsInt32 = reps.some((r) => {
    if (r.id !== "effect/schema/isBetween") return false;
    const p = r.payload as { minimum?: number; maximum?: number } | null;
    return (
      typeof p?.minimum === "number" &&
      typeof p?.maximum === "number" &&
      p.minimum >= INT32_MIN &&
      p.maximum <= INT32_MAX
    );
  });
  return fitsInt32 ? "int" : "long";
};

const representationId = (ast: AST.AST): string | undefined =>
  (ast.annotations as { representation?: { id?: string } } | undefined)?.representation?.id;

const descriptionOf = (ast: AST.AST): string | undefined => {
  const own = (ast.annotations as { description?: unknown } | undefined)?.description;
  if (typeof own === "string") return own;
  if (AST.isUnion(ast)) {
    for (const member of ast.types) {
      const d = (member.annotations as { description?: unknown } | undefined)?.description;
      if (typeof d === "string") return d;
    }
  }
  return undefined;
};

const NUMERIC_RANK: Record<string, number> = { int: 0, long: 1, double: 2 };

const columnTypeOf = (ast: AST.AST, path: string): { type: TableColumnType; nullable: boolean } => {
  switch (ast._tag) {
    case "Suspend":
      return columnTypeOf(ast.thunk(), path);
    case "Union": {
      const members = ast.types.filter(
        (t) => !(AST.isNull(t) || AST.isUndefined(t) || AST.isVoid(t)),
      );
      const nullable = members.length !== ast.types.length;
      if (members.length === 0) throw unsupported(path, "always null");
      const resolved = members.map((m) => columnTypeOf(m, path));
      const anyNullable = nullable || resolved.some((r) => r.nullable);
      if (resolved.length === 1) return { type: resolved[0]!.type, nullable: anyNullable };
      const types = resolved.map((r) => r.type);
      if (types.every((t) => typeof t === "string" && t in NUMERIC_RANK)) {
        const widest = types.reduce((a, b) =>
          NUMERIC_RANK[a as string]! >= NUMERIC_RANK[b as string]! ? a : b,
        );
        return { type: widest, nullable: anyNullable };
      }
      if (types.every((t) => typeof t === "string" && t === types[0])) {
        return { type: types[0]!, nullable: anyNullable };
      }
      throw unsupported(path, "a union of different types");
    }
    case "Literal": {
      const literal = ast.literal;
      if (typeof literal === "string") return { type: "string", nullable: false };
      if (typeof literal === "boolean") return { type: "boolean", nullable: false };
      if (typeof literal === "bigint") return { type: "long", nullable: false };
      return { type: Number.isInteger(literal) ? "long" : "double", nullable: false };
    }
    case "Enum": {
      const values = ast.enums.map(([, value]) => value);
      if (values.every((v) => typeof v === "string")) return { type: "string", nullable: false };
      if (values.every((v) => typeof v === "number")) return { type: "long", nullable: false };
      throw unsupported(path, "an enum mixing strings and numbers");
    }
    case "TemplateLiteral":
      return { type: "string", nullable: false };
    case "String":
      return {
        type: hasCheck(checkReps(ast), "isUUID") ? "uuid" : "string",
        nullable: false,
      };
    case "Number":
      return { type: numberType(ast), nullable: false };
    case "Boolean":
      return { type: "boolean", nullable: false };
    case "BigInt":
      return { type: "long", nullable: false };
    case "Declaration": {
      const rep = representationId(ast);
      if (rep === "effect/schema/Date" || rep === "effect/schema/DateTimeUtc") {
        return { type: "timestamptz", nullable: false };
      }
      if (rep === "effect/schema/Uint8Array") return { type: "binary", nullable: false };
      if (rep === "effect/schema/ReadonlyMap" && ast.typeParameters.length === 2) {
        return {
          type: mapTypeOf(ast.typeParameters[0]!, ast.typeParameters[1]!, path),
          nullable: false,
        };
      }
      if (rep === "effect/schema/ReadonlySet" && ast.typeParameters.length === 1) {
        return { type: listTypeOf(ast.typeParameters[0]!, path), nullable: false };
      }
      // Schema.Class / TaggedClass: a declaration over the struct of its fields.
      const [inner] = ast.typeParameters;
      if (ast.typeParameters.length === 1 && inner !== undefined && AST.isObjects(inner)) {
        return columnTypeOf(inner, path);
      }
      throw unsupported(path, `a declaration (${rep ?? "unknown"})`);
    }
    case "Objects": {
      if (ast.propertySignatures.length === 0 && ast.indexSignatures.length === 1) {
        const index = ast.indexSignatures[0]!;
        return { type: mapTypeOf(index.parameter, index.type, path), nullable: false };
      }
      if (ast.indexSignatures.length > 0) {
        throw unsupported(path, "a struct with an index signature");
      }
      return { type: { type: "struct", fields: structFieldsOf(ast, path) }, nullable: false };
    }
    case "Arrays": {
      if (ast.elements.length === 0 && ast.rest.length === 1) {
        return { type: listTypeOf(ast.rest[0]!, path), nullable: false };
      }
      throw unsupported(path, "a tuple");
    }
    default:
      throw unsupported(path, `of schema type '${ast._tag}'`);
  }
};

const listTypeOf = (element: AST.AST, path: string): TableColumnType => {
  const resolved = columnTypeOf(element, `${path}.element`);
  return { type: "list", element: resolved.type, elementRequired: !resolved.nullable };
};

const mapTypeOf = (key: AST.AST, value: AST.AST, path: string): TableColumnType => {
  const k = columnTypeOf(key, `${path}.key`);
  if (k.nullable) throw unsupported(`${path}.key`, "a nullable map key");
  const v = columnTypeOf(value, `${path}.value`);
  return { type: "map", key: k.type, value: v.type, valueRequired: !v.nullable };
};

const structFieldsOf = (ast: AST.Objects, path: string): TableColumn[] =>
  ast.propertySignatures.map((property) => {
    if (typeof property.name !== "string") {
      throw unsupported(path, "keyed by a symbol");
    }
    const columnPath = path ? `${path}.${property.name}` : property.name;
    const resolved = columnTypeOf(property.type, columnPath);
    const optional = AST.isOptional(property.type);
    const keyDoc = (property.type.context?.annotations as { description?: unknown } | undefined)
      ?.description;
    const doc = typeof keyDoc === "string" ? keyDoc : descriptionOf(property.type);
    return {
      name: property.name,
      type: resolved.type,
      required: !(optional || resolved.nullable),
      ...(doc !== undefined ? { doc } : {}),
    };
  });

/**
 * Convert an Effect Schema struct (or `Schema.Class`) into the plain column
 * descriptor persisted in Table props. Plain descriptors pass through.
 */
export const toTableSchema = (schema: TableSchemaInput): TableSchema => {
  if (!S.isSchema(schema)) return schema as TableSchema;
  const root = columnTypeOf(schema.ast, "").type;
  if (typeof root === "string" || root.type !== "struct") {
    throw unsupported("", "not a struct (a table schema must be a Struct or Class)");
  }
  return { fields: root.fields };
};

// ---------------------------------------------------------------------------
// Normalization + comparison
// ---------------------------------------------------------------------------

/** Canonical primitive spelling: lowercase, no whitespace. */
const normalizePrimitive = (t: string): string => t.replaceAll(/\s+/g, "").toLowerCase();

/** Fill defaults so two descriptors compare structurally. */
export const normalizeColumnType = (t: TableColumnType): TableColumnType => {
  if (typeof t === "string") return normalizePrimitive(t) as TablePrimitiveType;
  switch (t.type) {
    case "struct":
      return { type: "struct", fields: normalizeColumns(t.fields) };
    case "list":
      return {
        type: "list",
        element: normalizeColumnType(t.element),
        elementRequired: t.elementRequired ?? false,
      };
    case "map":
      return {
        type: "map",
        key: normalizeColumnType(t.key),
        value: normalizeColumnType(t.value),
        valueRequired: t.valueRequired ?? false,
      };
  }
};

export const normalizeColumns = (fields: readonly TableColumn[]): TableColumn[] =>
  fields.map((f) => ({
    name: f.name,
    type: normalizeColumnType(f.type),
    required: f.required ?? false,
    ...(f.doc ? { doc: f.doc } : {}),
  }));

/** Iceberg wire struct fields → column descriptors. */
export const fromIcebergFields = (fields: readonly Iceberg.StructField[]): TableColumn[] =>
  fields.map((f) => ({
    name: f.name,
    type: fromIcebergType(f.type),
    required: f.required,
    ...(f.doc ? { doc: f.doc } : {}),
  }));

const fromIcebergType = (t: Iceberg.Type): TableColumnType => {
  if (typeof t === "string") return normalizePrimitive(t) as TablePrimitiveType;
  switch (t.type) {
    case "struct":
      return { type: "struct", fields: fromIcebergFields((t as Iceberg.StructType).fields) };
    case "list": {
      const l = t as Iceberg.ListType;
      return {
        type: "list",
        element: fromIcebergType(l.element),
        elementRequired: l.element_required,
      };
    }
    case "map": {
      const m = t as Iceberg.MapType;
      return {
        type: "map",
        key: fromIcebergType(m.key),
        value: fromIcebergType(m.value),
        valueRequired: m.value_required,
      };
    }
    default:
      return normalizePrimitive(t.type) as TablePrimitiveType;
  }
};

const decimalOf = (t: string) => /^decimal\((\d+),(\d+)\)$/.exec(t);

/** Iceberg's legal in-place type promotions. */
const isPromotion = (from: string, to: string): boolean => {
  if (from === to) return true;
  if (from === "int" && to === "long") return true;
  if (from === "float" && to === "double") return true;
  const a = decimalOf(from);
  const b = decimalOf(to);
  return !!a && !!b && a[2] === b[2] && Number(b[1]) >= Number(a[1]);
};

const describe = (t: TableColumnType): string =>
  typeof t === "string" ? t : t.type === "struct" ? "struct" : t.type;

/**
 * Compare an observed (or previously declared) schema with the desired one
 * and list every change Iceberg cannot apply without rewriting or losing
 * data: dropped columns, new required columns, required-ness tightened, and
 * type changes other than Iceberg's promotions (int→long, float→double,
 * decimal precision widening).
 */
export const destructiveChanges = (
  observed: readonly TableColumn[],
  desired: readonly TableColumn[],
): string[] => {
  const reasons: string[] = [];
  compareFields("", normalizeColumns(observed), normalizeColumns(desired), reasons);
  return reasons;
};

const compareFields = (
  prefix: string,
  observed: TableColumn[],
  desired: TableColumn[],
  reasons: string[],
) => {
  for (const o of observed) {
    const path = `${prefix}${o.name}`;
    const d = desired.find((f) => f.name === o.name);
    if (!d) {
      reasons.push(`drops column '${path}'`);
      continue;
    }
    if (!o.required && d.required) reasons.push(`makes optional column '${path}' required`);
    compareTypes(path, o.type, d.type, reasons);
  }
  for (const d of desired) {
    if (d.required && !observed.some((f) => f.name === d.name)) {
      reasons.push(
        `adds required column '${prefix}${d.name}' (new columns must be optional so existing rows stay valid)`,
      );
    }
  }
};

const compareTypes = (path: string, o: TableColumnType, d: TableColumnType, reasons: string[]) => {
  if (typeof o === "string" || typeof d === "string") {
    if (typeof o === "string" && typeof d === "string" && isPromotion(o, d)) return;
    reasons.push(`changes the type of '${path}' from ${describe(o)} to ${describe(d)}`);
    return;
  }
  if (o.type !== d.type) {
    reasons.push(`changes the type of '${path}' from ${describe(o)} to ${describe(d)}`);
    return;
  }
  if (o.type === "struct" && d.type === "struct") {
    compareFields(`${path}.`, o.fields, d.fields, reasons);
  } else if (o.type === "list" && d.type === "list") {
    if (!o.elementRequired && d.elementRequired) {
      reasons.push(`makes the elements of '${path}' required`);
    }
    compareTypes(`${path}.element`, o.element, d.element, reasons);
  } else if (o.type === "map" && d.type === "map") {
    if (JSON.stringify(o.key) !== JSON.stringify(d.key)) {
      reasons.push(`changes the key type of map '${path}'`);
    }
    if (!o.valueRequired && d.valueRequired) {
      reasons.push(`makes the values of '${path}' required`);
    }
    compareTypes(`${path}.value`, o.value, d.value, reasons);
  }
};

// ---------------------------------------------------------------------------
// Field ids
// ---------------------------------------------------------------------------

/**
 * Build the Iceberg wire schema for `desired`, reusing the field ids of
 * columns that already exist in `observed` (matched by name at the same
 * nesting) and assigning fresh ids from `nextId` to new columns. Ids at one
 * struct level are assigned before descending, like Iceberg's own clients.
 */
export const assignFieldIds = (
  desired: readonly TableColumn[],
  observed: readonly Iceberg.StructField[] | undefined,
  nextId: () => number,
): Iceberg.StructField[] => {
  const normalized = normalizeColumns(desired);
  const matches = normalized.map((f) => observed?.find((o) => o.name === f.name));
  const ids = matches.map((m) => m?.id ?? nextId());
  return normalized.map((f, i) => ({
    id: ids[i]!,
    name: f.name,
    required: f.required ?? false,
    type: buildType(f.type, matches[i]?.type, nextId),
    ...(f.doc ? { doc: f.doc } : {}),
  }));
};

const buildType = (
  t: TableColumnType,
  observed: Iceberg.Type | undefined,
  nextId: () => number,
): Iceberg.Type => {
  if (typeof t === "string") return t;
  const obs = typeof observed === "object" ? observed : undefined;
  switch (t.type) {
    case "struct":
      return {
        type: "struct",
        fields: assignFieldIds(
          t.fields,
          obs?.type === "struct" ? (obs as Iceberg.StructType).fields : undefined,
          nextId,
        ),
      };
    case "list": {
      const ol = obs?.type === "list" ? (obs as Iceberg.ListType) : undefined;
      const elementId = ol?.element_id ?? nextId();
      return {
        type: "list",
        element_id: elementId,
        element: buildType(t.element, ol?.element, nextId),
        element_required: t.elementRequired ?? false,
      };
    }
    case "map": {
      const om = obs?.type === "map" ? (obs as Iceberg.MapType) : undefined;
      const keyId = om?.key_id ?? nextId();
      const valueId = om?.value_id ?? nextId();
      return {
        type: "map",
        key_id: keyId,
        key: buildType(t.key, om?.key, nextId),
        value_id: valueId,
        value: buildType(t.value, om?.value, nextId),
        value_required: t.valueRequired ?? false,
      };
    }
  }
};

/** Highest field id anywhere in a wire schema. */
export const maxFieldId = (fields: readonly Iceberg.StructField[]): number => {
  let max = 0;
  const visitType = (t: Iceberg.Type) => {
    if (typeof t === "string") return;
    if (t.type === "struct") visitFields((t as Iceberg.StructType).fields);
    else if (t.type === "list") {
      const l = t as Iceberg.ListType;
      max = Math.max(max, l.element_id);
      visitType(l.element);
    } else if (t.type === "map") {
      const m = t as Iceberg.MapType;
      max = Math.max(max, m.key_id, m.value_id);
      visitType(m.key);
      visitType(m.value);
    }
  };
  const visitFields = (fs: readonly Iceberg.StructField[]) => {
    for (const f of fs) {
      max = Math.max(max, f.id);
      visitType(f.type);
    }
  };
  visitFields(fields);
  return max;
};

/** Dotted column path → field id, for partition source resolution. */
export const fieldIdsByPath = (fields: readonly Iceberg.StructField[]): Map<string, number> => {
  const out = new Map<string, number>();
  const visit = (prefix: string, fs: readonly Iceberg.StructField[]) => {
    for (const f of fs) {
      const path = `${prefix}${f.name}`;
      out.set(path, f.id);
      if (typeof f.type === "object" && f.type.type === "struct") {
        visit(`${path}.`, (f.type as Iceberg.StructType).fields);
      }
    }
  };
  visit("", fields);
  return out;
};

// ---------------------------------------------------------------------------
// Partition specs
// ---------------------------------------------------------------------------

export interface PartitionTerm {
  /** Dotted source column path. */
  column: string;
  /** Iceberg transform, e.g. `identity`, `day`, `bucket[16]`. */
  transform: string;
}

const UNARY: Record<string, string> = {
  identity: "identity",
  year: "year",
  years: "year",
  month: "month",
  months: "month",
  day: "day",
  days: "day",
  date: "day",
  hour: "hour",
  hours: "hour",
  void: "void",
};

const COLUMN = "([A-Za-z_][\\w]*(?:\\.[A-Za-z_][\\w]*)*)";
const UNARY_RE = new RegExp(`^(\\w+)\\(${COLUMN}\\)

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	.repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
	
		
); const BRACKET_RE = new RegExp(`^(bucket|truncate)\\[(\\d+)\\]\\(${COLUMN}\\) .repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
); const ARG_RE = new RegExp(`^(bucket|truncate)\\((\\d+),${COLUMN}\\) .repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
); const BARE_RE = new RegExp(`^${COLUMN} .repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
); /** * Parse a `partitionBy` term: `col`, `identity(col)`, `year|month|day|hour(col)`, * `bucket[N](col)` / `bucket(N, col)`, `truncate[N](col)` / `truncate(N, col)`. */ export const parsePartitionTerm = (term: string): PartitionTerm => { const t = term.replaceAll(/\s+/g, ""); let m = BRACKET_RE.exec(t) ?? ARG_RE.exec(t); if (m) return { column: m[3]!, transform: `${m[1]}[${m[2]}]` }; m = UNARY_RE.exec(t); if (m && UNARY[m[1]!.toLowerCase()]) { return { column: m[2]!, transform: UNARY[m[1]!.toLowerCase()]! }; } m = BARE_RE.exec(t); if (m) return { column: m[1]!, transform: "identity" }; throw new BasinTableSchemaUnsupported({ path: term, message: `Cloudflare.Basin.Table: cannot parse partition term '${term}' (expected e.g. 'day(at)', 'bucket[16](id)', 'region')`, }); }; export const partitionFieldName = ({ column, transform }: PartitionTerm): string => { const base = column.replaceAll(".", "_"); if (transform === "identity") return base; if (transform.startsWith("bucket")) return `${base}_bucket`; if (transform.startsWith("truncate")) return `${base}_trunc`; return `${base}_${transform}`; }; export const samePartitionTerms = (a: readonly PartitionTerm[], b: readonly PartitionTerm[]) => a.length === b.length && a.every((t, i) => t.column === b[i]!.column && t.transform === b[i]!.transform);
.repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
);\nconst BRACKET_RE = new RegExp(`^(bucket|truncate)\\\\[(\\\\d+)\\\\]\\\\(${COLUMN}\\\\) .repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
);\nconst ARG_RE = new RegExp(`^(bucket|truncate)\\\\((\\\\d+),${COLUMN}\\\\) .repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
);\nconst BARE_RE = new RegExp(`^${COLUMN} .repos/alchemy-effect/packages/cloudflare/src/Basin/internal/schema.ts at main · sailfork/t3code-github · Sailfork
);\n\n/**\n * Parse a `partitionBy` term: `col`, `identity(col)`, `year|month|day|hour(col)`,\n * `bucket[N](col)` / `bucket(N, col)`, `truncate[N](col)` / `truncate(N, col)`.\n */\nexport const parsePartitionTerm = (term: string): PartitionTerm => {\n const t = term.replaceAll(/\\s+/g, \"\");\n let m = BRACKET_RE.exec(t) ?? ARG_RE.exec(t);\n if (m) return { column: m[3]!, transform: `${m[1]}[${m[2]}]` };\n m = UNARY_RE.exec(t);\n if (m && UNARY[m[1]!.toLowerCase()]) {\n return { column: m[2]!, transform: UNARY[m[1]!.toLowerCase()]! };\n }\n m = BARE_RE.exec(t);\n if (m) return { column: m[1]!, transform: \"identity\" };\n throw new BasinTableSchemaUnsupported({\n path: term,\n message: `Cloudflare.Basin.Table: cannot parse partition term '${term}' (expected e.g. 'day(at)', 'bucket[16](id)', 'region')`,\n });\n};\n\nexport const partitionFieldName = ({ column, transform }: PartitionTerm): string => {\n const base = column.replaceAll(\".\", \"_\");\n if (transform === \"identity\") return base;\n if (transform.startsWith(\"bucket\")) return `${base}_bucket`;\n if (transform.startsWith(\"truncate\")) return `${base}_trunc`;\n return `${base}_${transform}`;\n};\n\nexport const samePartitionTerms = (a: readonly PartitionTerm[], b: readonly PartitionTerm[]) =>\n a.length === b.length &&\n a.every((t, i) => t.column === b[i]!.column && t.transform === b[i]!.transform);\n","preview":null,"history":null,"version":"5e48960d84b3f0aaf49886b16314c16410941520"},"/_session?shell=1":null}