sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ RDS/ Migrations.ts

import {
  inlineSqlParams,
  MigrationError,
  migrationError,
  runMigrations,
  type MigrationDialect,
  type NormalizedMigrationsInput,
  type SqlExecutor,
  type StampedMigrationsState,
} from "@alchemy.run/sql/Migrations";
import * as Lambda from "@distilled.cloud/aws/lambda";
import * as rds from "@distilled.cloud/aws/rds";
import * as rdsdata from "@distilled.cloud/aws/rds-data";
import * as secretsmanager from "@distilled.cloud/aws/secrets-manager";
import * as Effect from "effect/Effect";
import * as Redacted from "effect/Redacted";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";
import { withMigrationConnection, type MigrationConnectionTarget } from "./MigrationConnection.ts";
import type { MigrationRunnerRequest, MigrationRunnerResponse } from "./MigrationRunnerHandler.ts";

/**
 * How migrations reach an RDS database:
 *
 * - `"data-api"` — the RDS Data API over HTTPS. Aurora clusters with the
 *   Data API enabled only; no network route needed.
 * - `"direct"` — a `pg`/`mysql2` connection from the deploying machine to
 *   the database endpoint. Needs a network route (public endpoint, VPN,
 *   bastion).
 * - an `AWS.RDS.MigrationRunner` (or any `{ functionArn }`) — a Lambda in
 *   the database's VPC opens the connection; the deploying machine only
 *   invokes it.
 */
export type MigrationsTransport = "data-api" | "direct" | { functionArn: string };

const dialects: Record<string, "postgres" | "mysql" | undefined> = {
  postgres: "postgres",
  "aurora-postgresql": "postgres",
  mysql: "mysql",
  mariadb: "mysql",
  "aurora-mysql": "mysql",
};

/** The writer endpoint, database and credential sources of an instance. */
interface MigrationTarget {
  dialect: "postgres" | "mysql";
  host: string | undefined;
  port: number | undefined;
  database: string | undefined;
  /** The cluster ARN when the Data API is enabled on the instance's cluster. */
  dataApiArn: string | undefined;
  managedSecretArn: string | undefined;
}

const describeTarget = Effect.fn(function* (instance: rds.DBInstance) {
  const identifier = instance.DBInstanceIdentifier ?? "unknown";
  if (instance.DBClusterIdentifier) {
    const cluster = yield* rds
      .describeDBClusters({ DBClusterIdentifier: instance.DBClusterIdentifier })
      .pipe(
        Effect.map((response) => response.DBClusters?.[0]),
        Effect.catchTag("DBClusterNotFoundFault", () => Effect.undefined),
      );
    if (!cluster) {
      return yield* new MigrationError({
        message: `DB cluster '${instance.DBClusterIdentifier}' of instance '${identifier}' not found`,
      });
    }
    const engine = cluster.Engine ?? "";
    const dialect = dialects[engine];
    if (!dialect) {
      return yield* unsupportedEngine(instance.DBClusterIdentifier, engine);
    }
    return {
      dialect,
      host: cluster.Endpoint,
      port: cluster.Port,
      database: cluster.DatabaseName,
      dataApiArn: cluster.HttpEndpointEnabled ? cluster.DBClusterArn : undefined,
      managedSecretArn: cluster.MasterUserSecret?.SecretArn,
    } satisfies MigrationTarget;
  }
  const engine = instance.Engine ?? "";
  const dialect = dialects[engine];
  if (!dialect) {
    return yield* unsupportedEngine(identifier, engine);
  }
  return {
    dialect,
    host: instance.Endpoint?.Address,
    port: instance.Endpoint?.Port,
    database: instance.DBName,
    dataApiArn: undefined,
    managedSecretArn: instance.MasterUserSecret?.SecretArn,
  } satisfies MigrationTarget;
});

const unsupportedEngine = (identifier: string, engine: string) =>
  new MigrationError({
    message: `'${identifier}' runs ${engine || "an unknown engine"}; migrations support PostgreSQL, MySQL and MariaDB engines (including Aurora)`,
  });

// An auto-paused Serverless v2 writer resumes on the first request.
const resumeRetry = {
  while: (e: { readonly _tag: string }) => e._tag === "DatabaseResumingException",
  schedule: Schedule.spaced("5 seconds"),
  times: 10,
};

/**
 * The Data API as a {@link SqlExecutor}. Queries inline their params (the
 * Data API binds `:name` placeholders only), and batches run inside a
 * Data API transaction.
 */
const makeDataApiExecutor = Effect.fn(function* (options: {
  dialect: MigrationDialect;
  resourceArn: string;
  secretArn: string;
  database: string | undefined;
}) {
  const executeStatement = yield* rdsdata.executeStatement;
  const beginTransaction = yield* rdsdata.beginTransaction;
  const commitTransaction = yield* rdsdata.commitTransaction;
  const rollbackTransaction = yield* rdsdata.rollbackTransaction;
  const { dialect, resourceArn, secretArn, database } = options;

  return {
    dialect,
    query: (sql, params) =>
      executeStatement({
        resourceArn,
        secretArn,
        database,
        sql: inlineSqlParams(sql, params ?? [], dialect),
        formatRecordsAs: "JSON",
      }).pipe(
        Effect.retry(resumeRetry),
        Effect.mapError(migrationError("Data API query")),
        Effect.flatMap((result) =>
          Effect.try({
            try: () =>
              (result.formattedRecords ? JSON.parse(result.formattedRecords) : []) as Array<
                Record<string, unknown>
              >,
            catch: (cause) =>
              new MigrationError({
                message: `Data API returned malformed records: ${String(cause)}`,
                cause,
              }),
          }),
        ),
      ),
    batch: (statements) =>
      Effect.gen(function* () {
        const { transactionId } = yield* beginTransaction({
          resourceArn,
          secretArn,
          database,
        }).pipe(
          Effect.retry(resumeRetry),
          Effect.mapError(migrationError("Data API begin transaction")),
        );
        if (!transactionId) {
          return yield* new MigrationError({ message: "Data API returned no transaction id" });
        }
        yield* Effect.gen(function* () {
          yield* Effect.forEach(
            statements,
            (sql) => executeStatement({ resourceArn, secretArn, database, sql, transactionId }),
            { discard: true },
          );
          yield* commitTransaction({ resourceArn, secretArn, transactionId });
        }).pipe(
          Effect.onError(() =>
            rollbackTransaction({ resourceArn, secretArn, transactionId }).pipe(Effect.ignore),
          ),
          Effect.mapError(migrationError("Data API migration batch")),
        );
      }),
  } satisfies SqlExecutor;
});

/**
 * An in-VPC `AWS.RDS.MigrationRunner` as a {@link SqlExecutor}: every
 * query and every migration batch is one synchronous invocation that opens
 * its own connection. Params are inlined so the payload is plain SQL.
 */
const makeRunnerExecutor = Effect.fn(function* (options: {
  functionArn: string;
  target: MigrationConnectionTarget;
}) {
  const invoke = yield* Lambda.invoke;
  const { functionArn, target } = options;
  const call = (request: MigrationRunnerRequest) =>
    Effect.gen(function* () {
      const response = yield* invoke({
        FunctionName: functionArn,
        Payload: JSON.stringify(request),
      }).pipe(Effect.mapError(migrationError(`Invoking migration runner ${functionArn}`)));
      const body = response.Payload
        ? yield* response.Payload.pipe(
            Stream.decodeText(),
            Stream.mkString,
            Effect.mapError(
              (cause) =>
                new MigrationError({
                  message: `Failed to read the migration runner's response: ${cause.message}`,
                  cause,
                }),
            ),
          )
        : "";
      const parsed = yield* Effect.try({
        try: () => JSON.parse(body) as MigrationRunnerResponse & { errorMessage?: string },
        catch: (cause) =>
          new MigrationError({
            message: `Migration runner returned a malformed response: ${body}`,
            cause,
          }),
      });
      if (response.FunctionError) {
        return yield* new MigrationError({
          message: `Migration runner ${functionArn} failed: ${parsed.errorMessage ?? body}`,
        });
      }
      if (!parsed.ok) {
        return yield* new MigrationError({ message: parsed.message });
      }
      return parsed.rows;
    });

  return {
    dialect: target.dialect,
    query: (sql, params) =>
      call({ op: "query", target, sql: inlineSqlParams(sql, params ?? [], target.dialect) }),
    batch: (statements) => call({ op: "batch", target, statements }).pipe(Effect.asVoid),
  } satisfies SqlExecutor;
});

const readSecretCredentials = Effect.fn(function* (secretArn: string) {
  const value = yield* secretsmanager
    .getSecretValue({ SecretId: secretArn })
    .pipe(Effect.mapError(migrationError(`Reading secret ${secretArn}`)));
  const secretString =
    value.SecretString === undefined
      ? "{}"
      : typeof value.SecretString === "string"
        ? value.SecretString
        : Redacted.value(value.SecretString);
  const secret = yield* Effect.try({
    try: () => JSON.parse(secretString) as { username?: string; password?: string },
    catch: (cause) => new MigrationError({ message: `Secret ${secretArn} is not JSON`, cause }),
  });
  if (!secret.username || !secret.password) {
    return yield* new MigrationError({
      message: `Secret ${secretArn} has no "username" and "password"`,
    });
  }
  return { username: secret.username, password: secret.password };
});

/**
 * Apply migrations to the database an RDS instance serves — its own, or its
 * Aurora cluster's when it is a cluster member. Call once the instance is
 * available. Credentials, in order: `secretArn`, then `credentials`
 * (direct and Lambda transports only), then the RDS-managed master secret.
 */
export const runDBInstanceMigrations = Effect.fn(function* (options: {
  instance: rds.DBInstance;
  input: NormalizedMigrationsInput;
  stamped: StampedMigrationsState;
  transport: MigrationsTransport | undefined;
  secretArn: string | undefined;
  credentials: { username: string; password: Redacted.Redacted<string> } | undefined;
}) {
  const identifier = options.instance.DBInstanceIdentifier ?? "unknown";
  const target = yield* describeTarget(options.instance);
  const transport = options.transport ?? (target.dataApiArn ? "data-api" : "direct");

  if (transport === "data-api") {
    const resourceArn = target.dataApiArn;
    if (!resourceArn) {
      return yield* new MigrationError({
        message: options.instance.DBClusterIdentifier
          ? `DB cluster '${options.instance.DBClusterIdentifier}' does not have the Data API enabled (set enableHttpEndpoint: true)`
          : `DB instance '${identifier}' is not an Aurora cluster member; the Data API transport needs an Aurora cluster`,
      });
    }
    const secretArn = options.secretArn ?? target.managedSecretArn;
    if (!secretArn) {
      return yield* new MigrationError({
        message: `'${identifier}' has no RDS-managed master secret; pass migrationsSecretArn`,
      });
    }
    return yield* runMigrations({
      input: options.input,
      stamped: options.stamped,
      withExecutor: (apply) =>
        makeDataApiExecutor({
          dialect: target.dialect,
          resourceArn,
          secretArn,
          database: target.database,
        }).pipe(Effect.flatMap(apply)),
    });
  }

  if (!target.host || !target.port) {
    return yield* new MigrationError({ message: `'${identifier}' has no endpoint yet` });
  }
  if (target.dialect === "mysql" && !target.database) {
    return yield* new MigrationError({
      message: `'${identifier}' has no database name; set dbName (or databaseName on the cluster) to apply migrations`,
    });
  }
  const secretArn =
    options.secretArn ?? (options.credentials ? undefined : target.managedSecretArn);
  const credentials = secretArn
    ? yield* readSecretCredentials(secretArn)
    : options.credentials
      ? {
          username: options.credentials.username,
          password: Redacted.value(options.credentials.password),
        }
      : undefined;
  if (!credentials) {
    return yield* new MigrationError({
      message: `'${identifier}' has no credentials for migrations; set manageMasterUserPassword, masterUserPassword, or migrationsSecretArn`,
    });
  }
  const connection: MigrationConnectionTarget = {
    dialect: target.dialect,
    host: target.host,
    port: target.port,
    // PostgreSQL always has the `postgres` database.
    database: target.database ?? (target.dialect === "postgres" ? "postgres" : undefined),
    ...credentials,
  };

  return yield* runMigrations({
    input: options.input,
    stamped: options.stamped,
    withExecutor: (apply) =>
      transport === "direct"
        ? withMigrationConnection(connection, apply)
        : makeRunnerExecutor({ functionArn: transport.functionArn, target: connection }).pipe(
            Effect.flatMap(apply),
          ),
  });
});