sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Bedrock/ DataSource.ts

import * as bedrock from "@distilled.cloud/aws/bedrock-agent";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";

/**
 * The connector configuration for a data source. Passed through to the
 * Bedrock API unchanged — see the AWS SDK `DataSourceConfiguration` shape.
 * The common case is a `S3` type with `s3Configuration.bucketArn` set.
 */
export type DataSourceConfiguration = bedrock.DataSourceConfiguration;

/**
 * Controls document chunking, parsing, and custom transformation before
 * embedding. Passed through unchanged — see the AWS SDK
 * `VectorIngestionConfiguration` shape.
 */
export type VectorIngestionConfiguration = bedrock.VectorIngestionConfiguration;

/**
 * What happens to the vector embeddings when the data source is deleted.
 * `DELETE` removes them; `RETAIN` leaves them in the vector store.
 */
export type DataDeletionPolicy = "RETAIN" | "DELETE";

export interface DataSourceProps {
  /**
   * The id of the {@link KnowledgeBase} this data source feeds. Accepts a
   * knowledge base's `knowledgeBaseId` output. Changing the knowledge base
   * triggers a replacement.
   */
  knowledgeBaseId: string;
  /**
   * Name of the data source (1-100 characters). If omitted, a deterministic
   * physical name is generated from the app, stage, and logical ID. Changing
   * the name triggers a replacement.
   */
  name?: string;
  /**
   * A description of the data source.
   */
  description?: string;
  /**
   * The connector configuration (S3 bucket, web crawler, Confluence, ...).
   */
  dataSourceConfiguration: DataSourceConfiguration;
  /**
   * What happens to vector embeddings when the data source is deleted.
   * @default "RETAIN"
   */
  dataDeletionPolicy?: DataDeletionPolicy;
  /**
   * Document chunking / parsing configuration applied before embedding.
   */
  vectorIngestionConfiguration?: VectorIngestionConfiguration;
}

export interface DataSource extends Resource<
  "AWS.Bedrock.DataSource",
  DataSourceProps,
  {
    /**
     * The unique identifier of the knowledge base the data source belongs to.
     */
    knowledgeBaseId: string;
    /**
     * The unique identifier of the data source.
     */
    dataSourceId: string;
    /**
     * Name of the data source.
     */
    name: string;
  }
> {}

/**
 * A data source attached to an Amazon Bedrock {@link KnowledgeBase} — the
 * origin of the documents the knowledge base embeds and indexes.
 *
 * The most common source is an S3 bucket. After the data source is created,
 * start an ingestion job (`bedrock-agent:StartIngestionJob`) to crawl the
 * source, chunk + embed the documents, and write them to the vector store.
 * Ingestion is not part of the desired-state lifecycle — trigger it whenever
 * the underlying documents change.
 *
 * ### Creating Data Sources
 * **Example:** S3 Data Source
 * ```typescript
 * import * as Bedrock from "@alchemy.run/aws/Bedrock";
 *
 * const source = yield* Bedrock.DataSource("docs-bucket", {
 *   knowledgeBaseId: kb.knowledgeBaseId,
 *   dataSourceConfiguration: {
 *     type: "S3",
 *     s3Configuration: { bucketArn: bucket.bucketArn },
 *   },
 *   dataDeletionPolicy: "DELETE",
 * });
 * ```
 *
 * @resource
 */
export const DataSource = Resource<DataSource>("AWS.Bedrock.DataSource");

/** Data-source status values indicating an in-flight transition. */
const DS_TRANSIENT = new Set(["CREATING", "UPDATING", "DELETING"]);

/**
 * A data source rejects deletion with a `ConflictException` while the parent
 * knowledge base is busy. Bounded retry (~60s), wrapped in an explicitly-typed
 * helper so the `Effect.retry` return type does not widen the provider layer's
 * requirement to `unknown` (see PATTERNS §7).
 */
const retryWhileConflict = <A, E extends { _tag: string }, R>(
  self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
  Effect.retry(self, {
    while: (e) => e._tag === "ConflictException",
    schedule: Schedule.max([Schedule.fixed("3 seconds"), Schedule.recurs(20)]),
  });

export const DataSourceProvider = () =>
  Provider.effect(
    DataSource,
    Effect.gen(function* () {
      const createName = Effect.fn(function* (id: string, props: Pick<DataSourceProps, "name">) {
        return props.name ?? (yield* createPhysicalName({ id, maxLength: 100 }));
      });

      const getDsOrUndefined = Effect.fn(function* (knowledgeBaseId: string, dataSourceId: string) {
        return yield* bedrock.getDataSource({ knowledgeBaseId, dataSourceId }).pipe(
          Effect.map((r) => r.dataSource),
          Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
        );
      });

      const waitForSettled = Effect.fn(function* (knowledgeBaseId: string, dataSourceId: string) {
        return yield* bedrock.getDataSource({ knowledgeBaseId, dataSourceId }).pipe(
          Effect.map((r) => r.dataSource),
          Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
          Effect.repeat({
            schedule: Schedule.fixed("3 seconds"),
            until: (ds) => ds === undefined || !DS_TRANSIENT.has(ds.status),
            times: 40,
          }),
        );
      });

      return DataSource.Provider.of({
        stables: ["knowledgeBaseId", "dataSourceId", "name"],

        // Data sources are scoped to a parent knowledge base — enumeration
        // requires the KB id, so this returns empty (the engine keys off state).
        list: () => Effect.succeed([]),

        read: Effect.fn(function* ({ olds, output }) {
          const knowledgeBaseId = output?.knowledgeBaseId ?? olds?.knowledgeBaseId;
          const dataSourceId = output?.dataSourceId;
          if (knowledgeBaseId === undefined || dataSourceId === undefined) {
            return undefined;
          }
          const ds = yield* getDsOrUndefined(knowledgeBaseId, dataSourceId);
          if (ds === undefined || ds.status === "DELETING") return undefined;
          return {
            knowledgeBaseId: ds.knowledgeBaseId,
            dataSourceId: ds.dataSourceId,
            name: ds.name,
          };
        }),

        diff: Effect.fn(function* ({ id, news, olds }) {
          if (!isResolved(news)) return undefined;
          if ((olds?.knowledgeBaseId ?? undefined) !== (news?.knowledgeBaseId ?? undefined)) {
            return { action: "replace" } as const;
          }
          const oldName = yield* createName(id, olds ?? ({} as DataSourceProps));
          const newName = yield* createName(id, news ?? ({} as DataSourceProps));
          if (oldName !== newName) {
            return { action: "replace" } as const;
          }
          // description, config, and deletion policy converge via update.
        }),

        reconcile: Effect.fn(function* ({ id, news = {} as DataSourceProps, output, session }) {
          const name = output?.name ?? (yield* createName(id, news));
          const knowledgeBaseId = news.knowledgeBaseId;

          // 1. OBSERVE
          let ds = output?.dataSourceId
            ? yield* getDsOrUndefined(knowledgeBaseId, output.dataSourceId)
            : undefined;

          if (ds === undefined) {
            // 2. ENSURE
            const created = yield* bedrock.createDataSource({
              knowledgeBaseId,
              name,
              description: news.description,
              dataSourceConfiguration: news.dataSourceConfiguration,
              dataDeletionPolicy: news.dataDeletionPolicy,
              vectorIngestionConfiguration: news.vectorIngestionConfiguration,
            });
            ds = created.dataSource;
            ds = (yield* waitForSettled(knowledgeBaseId, ds.dataSourceId)) ?? ds;
          } else {
            // 3. SYNC
            ds = (yield* waitForSettled(knowledgeBaseId, ds.dataSourceId)) ?? ds;
            const drifted =
              ds.description !== news.description ||
              JSON.stringify(ds.dataSourceConfiguration) !==
                JSON.stringify(news.dataSourceConfiguration) ||
              (ds.dataDeletionPolicy ?? undefined) !== (news.dataDeletionPolicy ?? undefined) ||
              JSON.stringify(ds.vectorIngestionConfiguration ?? null) !==
                JSON.stringify(news.vectorIngestionConfiguration ?? null);
            if (drifted) {
              yield* bedrock.updateDataSource({
                knowledgeBaseId,
                dataSourceId: ds.dataSourceId,
                name,
                description: news.description,
                dataSourceConfiguration: news.dataSourceConfiguration,
                dataDeletionPolicy: news.dataDeletionPolicy,
                vectorIngestionConfiguration: news.vectorIngestionConfiguration,
              });
              ds = (yield* waitForSettled(knowledgeBaseId, ds.dataSourceId)) ?? ds;
            }
          }

          yield* session.note(ds.dataSourceId);
          return {
            knowledgeBaseId,
            dataSourceId: ds.dataSourceId,
            name,
          };
        }),

        delete: Effect.fn(function* ({ output }) {
          yield* retryWhileConflict(
            bedrock.deleteDataSource({
              knowledgeBaseId: output.knowledgeBaseId,
              dataSourceId: output.dataSourceId,
            }),
          ).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
        }),
      });
    }),
  );