sailfork / t3code-github / .repos/ alchemy-effect/ packages/ gcp/ src/ Dataplex/ DataProduct.ts

import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import { tagRecord } from "alchemy/Tags";
import * as Effect from "effect/Effect";
import * as Schedule from "effect/Schedule";
import { GcpEnvironment } from "../Environment.ts";
import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
import type { Providers } from "../Providers.ts";
import {
  DataplexNotResolved,
  collectPages,
  fingerprint,
  hasAlchemyLabelMap,
  listAtLocation,
  normalizeLocation,
  parentOf,
  parseName,
  replaceOnIdentity,
  retryQuota,
  toPhysicalId,
  userLabels,
  waitForOperation,
  waitUntilExists,
  waitUntilGone,
} from "./internal.ts";

export type DataProductAccessGroup = dataplex.GoogleCloudDataplexV1DataProductAccessGroup;

export type DataProductAccessApprovalConfig =
  dataplex.GoogleCloudDataplexV1DataProductAccessApprovalConfig;

export type DataProductProps = {
  /**
   * Data product id. If omitted, a unique name is generated. Must match
   * `^[a-z]([a-z0-9-]{0,61}[a-z0-9])?

	
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
		
	.repos/alchemy-effect/packages/gcp/src/Dataplex/DataProduct.ts at main · sailfork/t3code-github · Sailfork
	
		
. Immutable — changing it * replaces the product. */ dataProductId?: string; /** * Region (`us-central1`, …). Immutable — changing it replaces the * product. * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`) */ location?: string; /** * User-friendly display name. Required. */ displayName: string; /** * Description of the data product. */ description?: string; /** * User labels. Alchemy ownership labels are merged in automatically. */ labels?: Record<string, string>; /** * Emails of the data product owners. Required. */ ownerEmails: string[]; /** * Base64-encoded icon (max 3 MiB). */ icon?: string; /** * Access groups keyed by group id. */ accessGroups?: Record<string, DataProductAccessGroup>; /** * Access-approval configuration. */ accessApprovalConfig?: DataProductAccessApprovalConfig; }; export type DataProduct = Resource< "GCP.Dataplex.DataProduct", DataProductProps, { /** Full resource name. */ name: string; /** Data product id (last path segment). */ dataProductId: string; /** Project id. */ project: string; /** Location id. */ location: string; /** Display name. */ displayName: string | undefined; /** Description. */ description: string | undefined; /** User labels (Alchemy ownership labels stripped). */ labels: Record<string, string>; /** Owner emails. */ ownerEmails: string[]; /** Icon payload. */ icon: string | undefined; /** Access groups. */ accessGroups: Record<string, DataProductAccessGroup | undefined> | undefined; /** Access-approval configuration. */ accessApprovalConfig: DataProductAccessApprovalConfig | undefined; /** Number of attached data assets. */ assetCount: number | undefined; /** Server etag. */ etag: string | undefined; /** System uid. */ uid: string | undefined; /** RFC3339 creation timestamp. */ createTime: string | undefined; /** RFC3339 last-update timestamp. */ updateTime: string | undefined; }, never, Providers >; /** * A Dataplex Data Product — a curated collection of data assets packaged * for a use case. * * Location and product id are immutable. Display name, description, * labels, owners, icon, access groups, and approval config update in * place. * * ### Creating a Data Product * **Example:** Product with an owner * ```typescript * const product = yield* GCP.Dataplex.DataProduct("Sales", { * displayName: "Sales mart", * ownerEmails: ["owner@example.com"], * labels: { env: "test" }, * }); * ``` * * @resource * @category Dataplex */ export const DataProduct = Resource<DataProduct>("GCP.Dataplex.DataProduct"); const resourceName = (project: string, location: string, dataProductId: string) => `projects/${project}/locations/${location}/dataProducts/${dataProductId}`; const toAttrs = (product: dataplex.GoogleCloudDataplexV1DataProduct, project: string) => { const name = product.name ?? ""; const parsed = parseName(name, "dataProducts"); return { name, dataProductId: parsed.id, project: parsed.project || project, location: parsed.location, displayName: product.displayName, description: product.description, labels: userLabels(product.labels), ownerEmails: [...(product.ownerEmails ?? [])], icon: product.icon, accessGroups: product.accessGroups, accessApprovalConfig: product.accessApprovalConfig, assetCount: product.assetCount, etag: product.etag, uid: product.uid, createTime: product.createTime, updateTime: product.updateTime, }; }; const getByName = (name: string) => retryQuota(dataplex.getProjectsLocationsDataProducts({ name })).pipe( Effect.catchTag("NotFound", () => Effect.undefined), ); const listProducts = (project: string, region: string) => { const collect = (parent: string) => collectPages( dataplex.listProjectsLocationsDataProducts.pages({ parent, pageSize: 1000, }), (page) => page.dataProducts, ).pipe( Effect.map((items) => items.filter((item) => hasAlchemyLabelMap(item.labels))), Effect.catchTag("NotFound", () => Effect.succeed([])), ); return listAtLocation(project, region, collect); }; export const listAlchemyDataProducts = (project: string, region: string) => listProducts(project, region); export const DataProductProvider = () => Provider.succeed(DataProduct, { stables: ["name", "dataProductId", "project", "location", "uid", "createTime"], diff: Effect.fn(function* ({ news, olds, output }) { if (!isResolved(news)) return undefined; const env = yield* GcpEnvironment.current; return replaceOnIdentity({ previousId: olds?.dataProductId ?? output?.dataProductId, nextId: news.dataProductId ?? olds?.dataProductId ?? output?.dataProductId, previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region), nextLocation: normalizeLocation( news.location ?? olds?.location ?? output?.location, env.region, ), }); }), read: Effect.fn(function* ({ id, olds, output }) { const env = yield* GcpEnvironment.current; const dataProductId = yield* toPhysicalId( id, olds?.dataProductId, output?.dataProductId, "dataproduct", ); const location = normalizeLocation(olds?.location ?? output?.location, env.region); const name = output?.name ?? resourceName(env.project, location, dataProductId); const existing = yield* getByName(name); if (existing === undefined) return undefined; const attrs = toAttrs(existing, env.project); return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs); }), list: () => Effect.gen(function* () { const env = yield* GcpEnvironment.current; const items = yield* listProducts(env.project, env.region); return items.map((item) => toAttrs(item, env.project)); }), reconcile: Effect.fn(function* ({ id, news, output }) { const env = yield* GcpEnvironment.current; const dataProductId = yield* toPhysicalId( id, news.dataProductId, output?.dataProductId, "dataproduct", ); const location = normalizeLocation(news.location ?? output?.location, env.region); const name = resourceName(env.project, location, dataProductId); const desiredLabels = { ...toLabels(news.labels), ...(yield* createInternalLabels(id)), }; let current = yield* getByName(output?.name ?? name); if (current === undefined) { const created = yield* retryQuota( dataplex.createProjectsLocationsDataProducts({ parent: parentOf(env.project, location), dataProductId, body: { displayName: news.displayName, description: news.description, labels: desiredLabels, ownerEmails: news.ownerEmails, icon: news.icon, accessGroups: news.accessGroups, accessApprovalConfig: news.accessApprovalConfig, }, }), ).pipe(Effect.catchTag("Conflict", () => Effect.undefined)); if (created !== undefined) { yield* waitForOperation(created); } current = yield* waitUntilExists(getByName(name), name); } if (current === undefined) { return yield* new DataplexNotResolved({ resourceName: name }); } const observedLabels = tagRecord(current.labels); const { upsert, removed } = diffLabels(observedLabels, desiredLabels); const labelsChanged = upsert.length > 0 || removed.length > 0; const descriptionChanged = (current.description ?? "") !== (news.description ?? ""); const displayNameChanged = (current.displayName ?? "") !== news.displayName; const ownersChanged = fingerprint([...(current.ownerEmails ?? [])].sort()) !== fingerprint([...(news.ownerEmails ?? [])].sort()); const iconChanged = (current.icon ?? "") !== (news.icon ?? ""); const groupsChanged = fingerprint(current.accessGroups) !== fingerprint(news.accessGroups); const approvalChanged = fingerprint(current.accessApprovalConfig) !== fingerprint(news.accessApprovalConfig); if ( labelsChanged || descriptionChanged || displayNameChanged || ownersChanged || iconChanged || groupsChanged || approvalChanged ) { const operation = yield* dataplex.patchProjectsLocationsDataProducts({ name: current.name ?? name, updateMask: [ labelsChanged ? "labels" : undefined, descriptionChanged ? "description" : undefined, displayNameChanged ? "displayName" : undefined, ownersChanged ? "ownerEmails" : undefined, iconChanged ? "icon" : undefined, groupsChanged ? "accessGroups" : undefined, approvalChanged ? "accessApprovalConfig" : undefined, ] .filter((field): field is string => field !== undefined) .join(","), body: { name: current.name ?? name, etag: current.etag, displayName: news.displayName, description: news.description, labels: desiredLabels, ownerEmails: news.ownerEmails, icon: news.icon, accessGroups: news.accessGroups, accessApprovalConfig: news.accessApprovalConfig, }, }); yield* waitForOperation(operation); current = yield* waitUntilExists(getByName(current.name ?? name), current.name ?? name); } return toAttrs(current, env.project); }), delete: Effect.fn(function* ({ output }) { const operation = yield* dataplex .deleteProjectsLocationsDataProducts({ name: output.name, etag: output.etag, }) .pipe( Effect.retry({ while: (error) => error._tag === "Conflict", times: 8, schedule: Schedule.spaced("2 seconds"), }), Effect.catchTag("NotFound", () => Effect.undefined), ); if (operation !== undefined) { yield* waitForOperation(operation, { notFoundOk: true }); } yield* waitUntilGone(getByName(output.name), output.name); }), });
.repos/alchemy-effect/packages/gcp/src/Dataplex/DataProduct.ts at main · sailfork/t3code-github · Sailfork
. Immutable — changing it\n * replaces the product.\n */\n dataProductId?: string;\n /**\n * Region (`us-central1`, …). Immutable — changing it replaces the\n * product.\n * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)\n */\n location?: string;\n /**\n * User-friendly display name. Required.\n */\n displayName: string;\n /**\n * Description of the data product.\n */\n description?: string;\n /**\n * User labels. Alchemy ownership labels are merged in automatically.\n */\n labels?: Record\u003cstring, string>;\n /**\n * Emails of the data product owners. Required.\n */\n ownerEmails: string[];\n /**\n * Base64-encoded icon (max 3 MiB).\n */\n icon?: string;\n /**\n * Access groups keyed by group id.\n */\n accessGroups?: Record\u003cstring, DataProductAccessGroup>;\n /**\n * Access-approval configuration.\n */\n accessApprovalConfig?: DataProductAccessApprovalConfig;\n};\n\nexport type DataProduct = Resource\u003c\n \"GCP.Dataplex.DataProduct\",\n DataProductProps,\n {\n /** Full resource name. */\n name: string;\n /** Data product id (last path segment). */\n dataProductId: string;\n /** Project id. */\n project: string;\n /** Location id. */\n location: string;\n /** Display name. */\n displayName: string | undefined;\n /** Description. */\n description: string | undefined;\n /** User labels (Alchemy ownership labels stripped). */\n labels: Record\u003cstring, string>;\n /** Owner emails. */\n ownerEmails: string[];\n /** Icon payload. */\n icon: string | undefined;\n /** Access groups. */\n accessGroups: Record\u003cstring, DataProductAccessGroup | undefined> | undefined;\n /** Access-approval configuration. */\n accessApprovalConfig: DataProductAccessApprovalConfig | undefined;\n /** Number of attached data assets. */\n assetCount: number | undefined;\n /** Server etag. */\n etag: string | undefined;\n /** System uid. */\n uid: string | undefined;\n /** RFC3339 creation timestamp. */\n createTime: string | undefined;\n /** RFC3339 last-update timestamp. */\n updateTime: string | undefined;\n },\n never,\n Providers\n>;\n\n/**\n * A Dataplex Data Product — a curated collection of data assets packaged\n * for a use case.\n *\n * Location and product id are immutable. Display name, description,\n * labels, owners, icon, access groups, and approval config update in\n * place.\n *\n * ### Creating a Data Product\n * **Example:** Product with an owner\n * ```typescript\n * const product = yield* GCP.Dataplex.DataProduct(\"Sales\", {\n * displayName: \"Sales mart\",\n * ownerEmails: [\"owner@example.com\"],\n * labels: { env: \"test\" },\n * });\n * ```\n *\n * @resource\n * @category Dataplex\n */\nexport const DataProduct = Resource\u003cDataProduct>(\"GCP.Dataplex.DataProduct\");\n\nconst resourceName = (project: string, location: string, dataProductId: string) =>\n `projects/${project}/locations/${location}/dataProducts/${dataProductId}`;\n\nconst toAttrs = (product: dataplex.GoogleCloudDataplexV1DataProduct, project: string) => {\n const name = product.name ?? \"\";\n const parsed = parseName(name, \"dataProducts\");\n return {\n name,\n dataProductId: parsed.id,\n project: parsed.project || project,\n location: parsed.location,\n displayName: product.displayName,\n description: product.description,\n labels: userLabels(product.labels),\n ownerEmails: [...(product.ownerEmails ?? [])],\n icon: product.icon,\n accessGroups: product.accessGroups,\n accessApprovalConfig: product.accessApprovalConfig,\n assetCount: product.assetCount,\n etag: product.etag,\n uid: product.uid,\n createTime: product.createTime,\n updateTime: product.updateTime,\n };\n};\n\nconst getByName = (name: string) =>\n retryQuota(dataplex.getProjectsLocationsDataProducts({ name })).pipe(\n Effect.catchTag(\"NotFound\", () => Effect.undefined),\n );\n\nconst listProducts = (project: string, region: string) => {\n const collect = (parent: string) =>\n collectPages(\n dataplex.listProjectsLocationsDataProducts.pages({\n parent,\n pageSize: 1000,\n }),\n (page) => page.dataProducts,\n ).pipe(\n Effect.map((items) => items.filter((item) => hasAlchemyLabelMap(item.labels))),\n Effect.catchTag(\"NotFound\", () => Effect.succeed([])),\n );\n return listAtLocation(project, region, collect);\n};\n\nexport const listAlchemyDataProducts = (project: string, region: string) =>\n listProducts(project, region);\n\nexport const DataProductProvider = () =>\n Provider.succeed(DataProduct, {\n stables: [\"name\", \"dataProductId\", \"project\", \"location\", \"uid\", \"createTime\"],\n\n diff: Effect.fn(function* ({ news, olds, output }) {\n if (!isResolved(news)) return undefined;\n const env = yield* GcpEnvironment.current;\n return replaceOnIdentity({\n previousId: olds?.dataProductId ?? output?.dataProductId,\n nextId: news.dataProductId ?? olds?.dataProductId ?? output?.dataProductId,\n previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),\n nextLocation: normalizeLocation(\n news.location ?? olds?.location ?? output?.location,\n env.region,\n ),\n });\n }),\n\n read: Effect.fn(function* ({ id, olds, output }) {\n const env = yield* GcpEnvironment.current;\n const dataProductId = yield* toPhysicalId(\n id,\n olds?.dataProductId,\n output?.dataProductId,\n \"dataproduct\",\n );\n const location = normalizeLocation(olds?.location ?? output?.location, env.region);\n const name = output?.name ?? resourceName(env.project, location, dataProductId);\n const existing = yield* getByName(name);\n if (existing === undefined) return undefined;\n const attrs = toAttrs(existing, env.project);\n return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);\n }),\n\n list: () =>\n Effect.gen(function* () {\n const env = yield* GcpEnvironment.current;\n const items = yield* listProducts(env.project, env.region);\n return items.map((item) => toAttrs(item, env.project));\n }),\n\n reconcile: Effect.fn(function* ({ id, news, output }) {\n const env = yield* GcpEnvironment.current;\n const dataProductId = yield* toPhysicalId(\n id,\n news.dataProductId,\n output?.dataProductId,\n \"dataproduct\",\n );\n const location = normalizeLocation(news.location ?? output?.location, env.region);\n const name = resourceName(env.project, location, dataProductId);\n const desiredLabels = {\n ...toLabels(news.labels),\n ...(yield* createInternalLabels(id)),\n };\n\n let current = yield* getByName(output?.name ?? name);\n\n if (current === undefined) {\n const created = yield* retryQuota(\n dataplex.createProjectsLocationsDataProducts({\n parent: parentOf(env.project, location),\n dataProductId,\n body: {\n displayName: news.displayName,\n description: news.description,\n labels: desiredLabels,\n ownerEmails: news.ownerEmails,\n icon: news.icon,\n accessGroups: news.accessGroups,\n accessApprovalConfig: news.accessApprovalConfig,\n },\n }),\n ).pipe(Effect.catchTag(\"Conflict\", () => Effect.undefined));\n if (created !== undefined) {\n yield* waitForOperation(created);\n }\n current = yield* waitUntilExists(getByName(name), name);\n }\n\n if (current === undefined) {\n return yield* new DataplexNotResolved({ resourceName: name });\n }\n\n const observedLabels = tagRecord(current.labels);\n const { upsert, removed } = diffLabels(observedLabels, desiredLabels);\n const labelsChanged = upsert.length > 0 || removed.length > 0;\n const descriptionChanged = (current.description ?? \"\") !== (news.description ?? \"\");\n const displayNameChanged = (current.displayName ?? \"\") !== news.displayName;\n const ownersChanged =\n fingerprint([...(current.ownerEmails ?? [])].sort()) !==\n fingerprint([...(news.ownerEmails ?? [])].sort());\n const iconChanged = (current.icon ?? \"\") !== (news.icon ?? \"\");\n const groupsChanged = fingerprint(current.accessGroups) !== fingerprint(news.accessGroups);\n const approvalChanged =\n fingerprint(current.accessApprovalConfig) !== fingerprint(news.accessApprovalConfig);\n\n if (\n labelsChanged ||\n descriptionChanged ||\n displayNameChanged ||\n ownersChanged ||\n iconChanged ||\n groupsChanged ||\n approvalChanged\n ) {\n const operation = yield* dataplex.patchProjectsLocationsDataProducts({\n name: current.name ?? name,\n updateMask: [\n labelsChanged ? \"labels\" : undefined,\n descriptionChanged ? \"description\" : undefined,\n displayNameChanged ? \"displayName\" : undefined,\n ownersChanged ? \"ownerEmails\" : undefined,\n iconChanged ? \"icon\" : undefined,\n groupsChanged ? \"accessGroups\" : undefined,\n approvalChanged ? \"accessApprovalConfig\" : undefined,\n ]\n .filter((field): field is string => field !== undefined)\n .join(\",\"),\n body: {\n name: current.name ?? name,\n etag: current.etag,\n displayName: news.displayName,\n description: news.description,\n labels: desiredLabels,\n ownerEmails: news.ownerEmails,\n icon: news.icon,\n accessGroups: news.accessGroups,\n accessApprovalConfig: news.accessApprovalConfig,\n },\n });\n yield* waitForOperation(operation);\n current = yield* waitUntilExists(getByName(current.name ?? name), current.name ?? name);\n }\n\n return toAttrs(current, env.project);\n }),\n\n delete: Effect.fn(function* ({ output }) {\n const operation = yield* dataplex\n .deleteProjectsLocationsDataProducts({\n name: output.name,\n etag: output.etag,\n })\n .pipe(\n Effect.retry({\n while: (error) => error._tag === \"Conflict\",\n times: 8,\n schedule: Schedule.spaced(\"2 seconds\"),\n }),\n Effect.catchTag(\"NotFound\", () => Effect.undefined),\n );\n if (operation !== undefined) {\n yield* waitForOperation(operation, { notFoundOk: true });\n }\n yield* waitUntilGone(getByName(output.name), output.name);\n }),\n });\n","preview":null,"history":null,"version":"91662ccce1087e90ac105c59c8b8aadfd1b04973"},"/_session?shell=1":null}