sailfork / t3code-github / .repos/ alchemy-effect/ packages/ cloudflare/ src/ Pipelines/ StreamSchema.ts

import * as Data from "effect/Data";
import * as Schema from "effect/Schema";
import * as AST from "effect/SchemaAST";

/**
 * Scalar field types accepted by a structured stream schema.
 */
export type StreamFieldType =
  | "int32"
  | "int64"
  | "float32"
  | "float64"
  | "bool"
  | "string"
  | "binary"
  | "json";

/**
 * Timestamp precision of a `timestamp` stream field.
 */
export type StreamTimestampUnit = "second" | "millisecond" | "microsecond" | "nanosecond";

/**
 * The type of a stream field (or list element), without its name.
 */
export type StreamFieldItem =
  | {
      /** Scalar field type. */
      type: StreamFieldType;
      /** Whether the value must be present. */
      required?: boolean;
    }
  | {
      /** Timestamp field type. */
      type: "timestamp";
      /** Whether the value must be present. */
      required?: boolean;
      /**
       * Precision of numeric timestamps.
       * @default "millisecond"
       */
      unit?: StreamTimestampUnit;
    }
  | {
      /** List (array) field type. */
      type: "list";
      /** Whether the value must be present. */
      required?: boolean;
      /** Type of every element of the list. */
      items: StreamFieldItem;
    }
  | {
      /** Nested object field type. */
      type: "struct";
      /** Whether the value must be present. */
      required?: boolean;
      /** Fields of the nested object. */
      fields: StreamField[];
    };

/**
 * A single named field of a structured stream schema. `timestamp` fields
 * additionally accept a `unit`, `list` fields an `items` type and
 * `struct` fields nested `fields`.
 */
export type StreamField = StreamFieldItem & {
  /** Field name as it appears in ingested events. */
  name: string;
  /** Whether the field must be present in every event. */
  required?: boolean;
  /** Name to expose to SQL when it differs from `name`. */
  sqlName?: string;
  /** Metadata key the field is populated from instead of the event body. */
  metadataKey?: string;
};

/**
 * A structured stream schema as an explicit field list.
 */
export interface StreamFieldsSchema {
  /** Fields of the structured schema. */
  fields: StreamField[];
}

/**
 * Annotation key that overrides the stream field type derived from an
 * Effect Schema, e.g. to store a number as `int32` or `float32`:
 *
 * ```typescript
 * Schema.Number.annotate({ [StreamFieldTypeAnnotation]: "float32" })
 * ```
 */
export const StreamFieldTypeAnnotation = "pipelinesFieldType" as const;

/**
 * Raised when an Effect Schema passed as a stream `schema` has no
 * Pipelines representation (e.g. a union of different types).
 */
export class StreamSchemaUnsupported extends Data.TaggedError(
  "Cloudflare.Pipelines.StreamSchemaUnsupported",
)<{
  message: string;
  path: string;
}> {}

const unsupported = (path: string, what: string) =>
  new StreamSchemaUnsupported({
    path,
    message:
      `Cloudflare.Pipelines.Stream: field '${path || "<root>"}' is ${what}, ` +
      "which has no Pipelines stream field type. Use a field list or annotate the field " +
      `with { ${StreamFieldTypeAnnotation}: "json" }.`,
  });

const INT32_MIN = -2147483648;
const INT32_MAX = 2147483647;

interface CheckRep {
  id: string;
  payload: unknown;
}

const checkReps = (ast: AST.AST): CheckRep[] => {
  const out: CheckRep[] = [];
  const visit = (check: unknown) => {
    const c = check as {
      annotations?: { representation?: CheckRep };
      checks?: readonly unknown[];
    };
    if (c.annotations?.representation?.id) out.push(c.annotations.representation);
    c.checks?.forEach(visit);
  };
  ((ast as { checks?: readonly unknown[] }).checks ?? []).forEach(visit);
  return out;
};

const annotationOverride = (ast: AST.AST): StreamFieldType | "timestamp" | undefined => {
  const read = (annotations: unknown) =>
    (annotations as Record<string, unknown> | undefined)?.[StreamFieldTypeAnnotation];
  const candidates = [
    read((ast as { annotations?: unknown }).annotations),
    ...((ast as { checks?: readonly { annotations?: unknown }[] }).checks ?? []).map((c) =>
      read(c.annotations),
    ),
  ];
  const found = candidates.find((v) => typeof v === "string");
  return found as StreamFieldType | "timestamp" | undefined;
};

const numberType = (ast: AST.AST): StreamFieldType => {
  const reps = checkReps(ast);
  if (!reps.some((r) => r.id === "effect/schema/isInt")) return "float64";
  const fitsInt32 = reps.some((r) => {
    if (r.id !== "effect/schema/isBetween") return false;
    const p = r.payload as { minimum?: number; maximum?: number } | null;
    return (
      p?.minimum !== undefined &&
      p?.maximum !== undefined &&
      p.minimum >= INT32_MIN &&
      p.maximum <= INT32_MAX
    );
  });
  return fitsInt32 ? "int32" : "int64";
};

const isNullish = (ast: AST.AST) =>
  ast._tag === "Null" || ast._tag === "Undefined" || ast._tag === "Void";

const literalType = (value: unknown): StreamFieldType | undefined =>
  typeof value === "string"
    ? "string"
    : typeof value === "boolean"
      ? "bool"
      : typeof value === "number"
        ? Number.isInteger(value)
          ? "int64"
          : "float64"
        : undefined;

/**
 * Derive the stream field type of one (encoded-side) AST node. Returns the
 * item plus whether the value may be absent (nullable union members).
 */
const toItem = (ast: AST.AST, path: string): { item: StreamFieldItem; nullable: boolean } => {
  const override = annotationOverride(ast);
  if (override) {
    return { item: { type: override } as StreamFieldItem, nullable: false };
  }
  switch (ast._tag) {
    case "String":
    case "TemplateLiteral":
      return { item: { type: "string" }, nullable: false };
    case "Number":
      return { item: { type: numberType(ast) }, nullable: false };
    case "Boolean":
      return { item: { type: "bool" }, nullable: false };
    case "Literal": {
      const type = literalType(ast.literal);
      if (!type) throw unsupported(path, `the literal ${String(ast.literal)}`);
      return { item: { type }, nullable: false };
    }
    case "Enum": {
      const types = new Set(
        (ast as unknown as { enums: ReadonlyArray<readonly [string, unknown]> }).enums.map(
          ([, v]) => literalType(v),
        ),
      );
      const [type] = [...types];
      if (types.size !== 1 || !type) throw unsupported(path, "a mixed enum");
      return { item: { type }, nullable: false };
    }
    case "Unknown":
    case "Any":
    case "ObjectKeyword":
      return { item: { type: "json" }, nullable: false };
    case "Declaration": {
      const id = (ast.annotations?.representation as CheckRep | undefined)?.id;
      if (id === "effect/schema/Date") return { item: { type: "timestamp" }, nullable: false };
      if (id === "effect/schema/Uint8Array") return { item: { type: "binary" }, nullable: false };
      if (id === "effect/schema/Json") return { item: { type: "json" }, nullable: false };
      throw unsupported(path, `a ${AST.resolveIdentifier(ast) ?? "declared"} value`);
    }
    case "Arrays": {
      if (ast.elements.length > 0 || ast.rest.length !== 1) {
        throw unsupported(path, "a tuple");
      }
      const element = toItem(ast.rest[0]!, `${path}[]`);
      return {
        item: {
          type: "list",
          items: element.nullable ? { ...element.item, required: false } : element.item,
        },
        nullable: false,
      };
    }
    case "Objects": {
      if (ast.indexSignatures.length > 0 || ast.propertySignatures.length === 0) {
        // Records and empty objects carry arbitrary keys — store as JSON.
        return { item: { type: "json" }, nullable: false };
      }
      return {
        item: { type: "struct", fields: toFields(ast, path) },
        nullable: false,
      };
    }
    case "Union": {
      const members = ast.types.filter((t) => !isNullish(t));
      const nullable = members.length !== ast.types.length;
      if (members.length === 1) {
        return { item: toItem(members[0]!, path).item, nullable };
      }
      // A union of same-typed literals (e.g. `Schema.Literals(["a","b"])`).
      if (members.length > 0 && members.every((m) => m._tag === "Literal")) {
        const types = new Set(members.map((m) => literalType((m as AST.Literal).literal)));
        const [type] = [...types];
        if (types.size === 1 && type) return { item: { type }, nullable };
      }
      throw unsupported(path, "a union of different types");
    }
    case "Suspend":
      throw unsupported(path, "a recursive schema");
    default:
      throw unsupported(path, `of type ${ast._tag}`);
  }
};

const toFields = (ast: AST.Objects, path: string): StreamField[] =>
  ast.propertySignatures.map((ps) => {
    const name = String(ps.name);
    const fieldPath = path ? `${path}.${name}` : name;
    const { item, nullable } = toItem(ps.type, fieldPath);
    const optional = AST.isOptional(ps.type);
    return {
      ...item,
      name,
      required: !(optional || nullable),
    } as StreamField;
  });

/**
 * Returns true if `value` is an Effect Schema (rather than a field list).
 */
export const isEffectSchema = (value: unknown): value is Schema.Top => Schema.isSchema(value);

/**
 * Convert an Effect Schema (`Schema.Struct` / `Schema.Class`) into the
 * Pipelines stream field list. The schema's **encoded** side is used, so
 * transformations (e.g. `NumberFromString`) map to their wire type, and
 * records are sent with `Schema.toCodecJson(schema)`:
 *
 * - `String` → `string`, `Boolean` → `bool`
 * - `Number` → `float64`, `Int` → `int64` (`int32` when bounded to 32 bits)
 * - `Date` → `timestamp`, `Uint8Array` → `binary`
 * - `Array` → `list`, `Struct` / `Class` → `struct`
 * - `Record` / `Unknown` / `Json` → `json`
 * - optional or nullable fields → `required: false`
 *
 * Unions of different types, tuples and recursive schemas have no stream
 * representation and throw {@link StreamSchemaUnsupported}. Override any
 * field's type with the {@link StreamFieldTypeAnnotation} annotation.
 */
export const streamFieldsFromSchema = (schema: Schema.Top): StreamField[] => {
  const encoded = AST.toEncoded(schema.ast);
  if (encoded._tag !== "Objects" || encoded.propertySignatures.length === 0) {
    throw unsupported("", "not a Struct or Class with fields");
  }
  return toFields(encoded, "");
};

/**
 * Resolve a stream `schema` prop (field list or Effect Schema) to the
 * field-list form persisted in state and sent to Cloudflare.
 */
export const resolveStreamSchema = (
  schema: StreamFieldsSchema | Schema.Top | undefined,
): StreamFieldsSchema | undefined => {
  if (schema === undefined) return undefined;
  if (isEffectSchema(schema)) return { fields: streamFieldsFromSchema(schema) };
  return schema as StreamFieldsSchema;
};

/**
 * Canonical (default-filled, key-ordered) form of a field item, so
 * equivalent spellings compare equal.
 */
const canonicalItem = (item: StreamFieldItem): Record<string, unknown> => {
  const base: Record<string, unknown> = {
    type: item.type,
    required: item.required ?? false,
  };
  if (item.type === "timestamp") base.unit = item.unit ?? "millisecond";
  if (item.type === "list") base.items = canonicalItem(item.items);
  if (item.type === "struct") base.fields = canonicalStreamFields(item.fields);
  return base;
};

/**
 * Canonical form of a field list: defaults filled in (`required: false`,
 * `unit: "millisecond"`), absent optional keys dropped.
 */
export const canonicalStreamFields = (fields: ReadonlyArray<StreamField>): unknown[] =>
  fields.map((f) => {
    const out: Record<string, unknown> = { name: f.name, ...canonicalItem(f) };
    if (f.sqlName !== undefined) out.sqlName = f.sqlName;
    if (f.metadataKey !== undefined) out.metadataKey = f.metadataKey;
    return out;
  });