sailfork / t3code-github / .repos/ alchemy-effect/ packages/ aws/ src/ Kendra/ DataSource.ts

import * as kendra from "@distilled.cloud/aws/kendra";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import { createPhysicalName } from "alchemy/PhysicalName";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import * as EffectStream from "effect/Stream";
import { AWSEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";

export class KendraDataSourceInvalidProps extends Data.TaggedError("KendraDataSourceInvalidProps")<{
  readonly message: string;
  readonly cause?: unknown;
}> {}

export class KendraDataSourceUnexpectedResponse extends Data.TaggedError(
  "KendraDataSourceUnexpectedResponse",
)<{
  readonly message: string;
  readonly cause?: unknown;
}> {}

export type DataSourceStatus = kendra.DataSourceStatus;
export type DataSourceType = kendra.DataSourceType;

export interface DataSourceProps {
  /**
   * The identifier of the Kendra index the data source connects to.
   * Changing it replaces the data source.
   */
  indexId: string;
  /**
   * Name of the data source.
   * @default ${app}-${stage}-${id}
   */
  name?: string;
  /**
   * The connector type (e.g. `S3`, `TEMPLATE`, `WEBCRAWLER`, `CUSTOM`).
   * Changing it replaces the data source.
   */
  type: DataSourceType;
  /**
   * Connector-specific configuration (e.g. `S3Configuration` or
   * `TemplateConfiguration`). Required for all types except `CUSTOM`.
   */
  configuration?: kendra.DataSourceConfiguration;
  /**
   * VPC configuration for connectors that reach into a VPC.
   */
  vpcConfiguration?: kendra.DataSourceVpcConfiguration;
  /**
   * A description of the data source.
   */
  description?: string;
  /**
   * Sync schedule as a cron expression (e.g. `cron(0 12 * * ? *)`). When
   * omitted, syncs run only on demand.
   */
  schedule?: string;
  /**
   * ARN of the IAM role Kendra assumes to access the source content.
   * Required for all types except `CUSTOM`.
   */
  roleArn?: string;
  /**
   * The code for a language the source documents are in.
   * @default "en"
   */
  languageCode?: string;
  /**
   * Alter document metadata/content during ingestion.
   */
  customDocumentEnrichmentConfiguration?: kendra.CustomDocumentEnrichmentConfiguration;
  /**
   * Tags to associate with the data source.
   */
  tags?: Record<string, string>;
}

export interface DataSource extends Resource<
  "AWS.Kendra.DataSource",
  DataSourceProps,
  {
    /**
     * Service-assigned unique identifier of the data source (unique within
     * its index).
     */
    id: string;
    /**
     * The identifier of the index the data source belongs to.
     */
    indexId: string;
    /**
     * ARN of the data source.
     */
    arn: string;
    /**
     * The data source's name.
     */
    name: string;
    /**
     * The connector type.
     */
    type: DataSourceType | undefined;
    /**
     * Current lifecycle status of the data source.
     */
    status: DataSourceStatus | undefined;
    /**
     * Current tags reported for the data source.
     */
    tags: Record<string, string>;
  },
  never,
  Providers
> {}

/**
 * An Amazon Kendra data source — a connector that syncs documents from a
 * repository (S3 bucket, SharePoint, website, ...) into a Kendra index.
 *
 * ### Creating Data Sources
 * **Example:** S3 Data Source
 * ```typescript
 * import * as AWS from "@alchemy.run/aws";
 *
 * const source = yield* AWS.Kendra.DataSource("Docs", {
 *   indexId: index.id,
 *   type: "S3",
 *   roleArn: dataSourceRole.roleArn,
 *   configuration: {
 *     S3Configuration: {
 *       BucketName: bucket.bucketName,
 *     },
 *   },
 * });
 * ```
 *
 * **Example:** Scheduled Sync
 * ```typescript
 * const source = yield* AWS.Kendra.DataSource("Docs", {
 *   indexId: index.id,
 *   type: "S3",
 *   roleArn: dataSourceRole.roleArn,
 *   schedule: "cron(0 12 * * ? *)",
 *   configuration: {
 *     S3Configuration: { BucketName: bucket.bucketName },
 *   },
 * });
 * ```
 *
 * @resource
 */
export const DataSource = Resource<DataSource>("AWS.Kendra.DataSource");

const createDataSourceName = (id: string, props: { name?: string | undefined }) =>
  props.name ? Effect.succeed(props.name) : createPhysicalName({ id, maxLength: 100 });

const fetchTags = Effect.fn(function* (arn: string) {
  const response = yield* kendra
    .listTagsForResource({ ResourceARN: arn })
    .pipe(
      Effect.catchTag(
        ["ResourceNotFoundException", "ResourceUnavailableException"],
        () => Effect.undefined,
      ),
    );
  return Object.fromEntries((response?.Tags ?? []).map((tag) => [tag.Key, tag.Value]));
});

const currentArnOf = Effect.gen(function* () {
  const { accountId, region } = yield* AWSEnvironment.current;
  return (indexId: string, dataSourceId: string) =>
    `arn:aws:kendra:${region}:${accountId}:index/${indexId}/data-source/${dataSourceId}`;
});

interface DataSourceState {
  attrs: DataSource["Attributes"];
  described: kendra.DescribeDataSourceResponse;
}

const readDataSourceById = Effect.fn(function* (
  indexId: string,
  dataSourceId: string,
  arnOf: (indexId: string, dataSourceId: string) => string,
) {
  const described = yield* kendra
    .describeDataSource({ IndexId: indexId, Id: dataSourceId })
    .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
  if (!described || described.Status === "DELETING") return undefined;
  const arn = arnOf(described.IndexId ?? indexId, described.Id ?? dataSourceId);
  const state: DataSourceState = {
    described,
    attrs: {
      id: described.Id ?? dataSourceId,
      indexId: described.IndexId ?? indexId,
      arn,
      name: described.Name ?? "",
      type: described.Type,
      status: described.Status,
      tags: yield* fetchTags(arn),
    },
  };
  return state;
});

const findDataSourceByName = Effect.fn(function* (
  indexId: string,
  name: string,
  arnOf: (indexId: string, dataSourceId: string) => string,
) {
  const summaries = yield* kendra.listDataSources.pages({ IndexId: indexId }).pipe(
    EffectStream.runCollect,
    Effect.map((chunk) => Array.from(chunk).flatMap((page) => page.SummaryItems ?? [])),
    // The parent index may itself be gone.
    Effect.catchTag("ResourceNotFoundException", () =>
      Effect.succeed([] as kendra.DataSourceSummary[]),
    ),
  );
  const match = summaries.find((summary) => summary.Name === name && summary.Status !== "DELETING");
  if (!match?.Id) return undefined;
  return yield* readDataSourceById(indexId, match.Id, arnOf);
});

/**
 * A data source still transitioning toward the awaited status — retried by
 * {@link waitForDataSourceStatus}'s bounded schedule.
 */
class DataSourceNotReady extends Data.TaggedError("DataSourceNotReady")<{
  readonly id: string;
  readonly status: string | undefined;
}> {}

/**
 * A data source whose asynchronous provisioning converged to the terminal
 * `FAILED` status.
 */
export class DataSourceProvisioningFailed extends Data.TaggedError("DataSourceProvisioningFailed")<{
  readonly id: string;
  readonly message: string | undefined;
}> {}

// Explicitly-typed retry wrapper — an inline `Effect.retry` in provider
// lifecycle code leaks `Retry.Return`'s conditional type into declaration
// emit and widens the provider layer to `unknown` for every consumer of
// `AWS.providers()`.
const retryWhileNotReady = <A, E extends { readonly _tag: string }, R>(
  self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
  Effect.retry(self, {
    while: (e) => e._tag === "DataSourceNotReady",
    // Data source provisioning is usually well under a minute; budget ~5 min.
    schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(60)]),
  });

// CreateDataSource validates the IAM role up front; a freshly-created role
// may not have propagated yet and surfaces as ValidationException. Bounded
// retry through the propagation window (~60s). Explicitly typed for the same
// declaration-emit reason as retryWhileNotReady.
const retryThroughIamPropagation = <A, E extends { readonly _tag: string }, R>(
  self: Effect.Effect<A, E, R>,
): Effect.Effect<A, E, R> =>
  Effect.retry(self, {
    while: (e) => e._tag === "ValidationException",
    schedule: Schedule.max([Schedule.spaced("5 seconds"), Schedule.recurs(12)]),
  });

const waitForDataSourceStatus = (indexId: string, id: string, target: "ACTIVE" | "DELETED") =>
  retryWhileNotReady(
    Effect.gen(function* () {
      const described = yield* kendra
        .describeDataSource({ IndexId: indexId, Id: id })
        .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
      if (target === "DELETED") {
        if (described === undefined) return;
        return yield* new DataSourceNotReady({ id, status: described.Status });
      }
      if (described?.Status === "ACTIVE") return;
      if (described?.Status === "FAILED") {
        return yield* new DataSourceProvisioningFailed({
          id,
          message: described.ErrorMessage,
        });
      }
      return yield* new DataSourceNotReady({ id, status: described?.Status });
    }),
  );

export const DataSourceProvider = () =>
  Provider.effect(
    DataSource,
    Effect.gen(function* () {
      return {
        stables: ["id", "indexId", "arn"],
        // Keyed by a parent index; cannot be enumerated account-wide without
        // iterating every index — treated as a sub-resource per the factory
        // list() convention.
        list: () => Effect.succeed([]),
        read: Effect.fn(function* ({ id, olds, output }) {
          const arnOf = yield* currentArnOf;
          const indexId = output?.indexId ?? olds?.indexId;
          if (indexId === undefined) return undefined;
          const state = output?.id
            ? yield* readDataSourceById(indexId, output.id, arnOf)
            : yield* findDataSourceByName(
                indexId,
                yield* createDataSourceName(id, olds ?? {}),
                arnOf,
              );
          if (!state) return undefined;
          return (yield* hasAlchemyTags(id, state.attrs.tags as Tags))
            ? state.attrs
            : Unowned(state.attrs);
        }),
        diff: Effect.fn(function* ({ news, olds }) {
          if (!isResolved(news)) return;
          if (olds === undefined) return;
          // The parent index and connector type are fixed at creation.
          if (olds.indexId !== news.indexId || olds.type !== news.type) {
            return { action: "replace" } as const;
          }
        }),
        reconcile: Effect.fn(function* ({ id, news, output, session }) {
          if (!news) {
            return yield* new KendraDataSourceInvalidProps({
              message: "Kendra DataSource requires props",
            });
          }
          const arnOf = yield* currentArnOf;
          const indexId = news.indexId;
          const name = yield* createDataSourceName(id, news);
          const internalTags = yield* createInternalTags(id);
          const desiredTags = { ...internalTags, ...news.tags };

          // Observe — prefer the cached id; fall back to a name lookup so a
          // create whose state failed to persist is adopted, not duplicated.
          let state = output?.id
            ? yield* readDataSourceById(indexId, output.id, arnOf)
            : yield* findDataSourceByName(indexId, name, arnOf);

          // Ensure — create if missing, then wait for ACTIVE.
          if (state === undefined) {
            const created = yield* retryThroughIamPropagation(
              kendra.createDataSource({
                IndexId: indexId,
                Name: name,
                Type: news.type,
                Configuration: news.configuration,
                VpcConfiguration: news.vpcConfiguration,
                Description: news.description,
                Schedule: news.schedule,
                RoleArn: news.roleArn,
                LanguageCode: news.languageCode,
                CustomDocumentEnrichmentConfiguration: news.customDocumentEnrichmentConfiguration,
                Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
                  Key,
                  Value,
                })),
              }),
            );
            yield* session.note(`Creating data source ${name} (${created.Id})...`);
            yield* waitForDataSourceStatus(indexId, created.Id, "ACTIVE");
            state = yield* readDataSourceById(indexId, created.Id, arnOf);
            if (state === undefined) {
              return yield* new KendraDataSourceUnexpectedResponse({
                message: `failed to read created data source ${name}`,
              });
            }
          }

          // Sync mutable settings via UpdateDataSource — only when drifted.
          const described = state.described;
          const needsUpdate =
            name !== described.Name ||
            (news.description ?? "") !== (described.Description ?? "") ||
            (news.schedule ?? "") !== (described.Schedule ?? "") ||
            (news.roleArn !== undefined && news.roleArn !== described.RoleArn) ||
            (news.languageCode !== undefined && news.languageCode !== described.LanguageCode) ||
            news.configuration !== undefined ||
            news.vpcConfiguration !== undefined ||
            news.customDocumentEnrichmentConfiguration !== undefined;
          if (needsUpdate) {
            yield* kendra.updateDataSource({
              Id: state.attrs.id,
              IndexId: indexId,
              Name: name,
              Configuration: news.configuration,
              VpcConfiguration: news.vpcConfiguration,
              Description: news.description,
              Schedule: news.schedule,
              RoleArn: news.roleArn,
              LanguageCode: news.languageCode,
              CustomDocumentEnrichmentConfiguration: news.customDocumentEnrichmentConfiguration,
            });
            yield* waitForDataSourceStatus(indexId, state.attrs.id, "ACTIVE");
            yield* session.note(`Updated data source ${name}`);
          }

          // Sync tags — diff against observed cloud tags.
          const { removed, upsert } = diffTags(state.attrs.tags, desiredTags);
          if (removed.length > 0) {
            yield* kendra.untagResource({
              ResourceARN: state.attrs.arn,
              TagKeys: removed,
            });
          }
          if (upsert.length > 0) {
            yield* kendra.tagResource({
              ResourceARN: state.attrs.arn,
              Tags: upsert.map(({ Key, Value }) => ({ Key, Value })),
            });
          }

          yield* session.note(state.attrs.arn);

          const final = yield* readDataSourceById(indexId, state.attrs.id, arnOf);
          if (!final) {
            return yield* new KendraDataSourceUnexpectedResponse({
              message: `failed to read reconciled data source ${name}`,
            });
          }
          return final.attrs;
        }),
        delete: Effect.fn(function* ({ output }) {
          yield* kendra
            .deleteDataSource({ IndexId: output.indexId, Id: output.id })
            .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
          yield* waitForDataSourceStatus(output.indexId, output.id, "DELETED");
        }),
      };
    }),
  );