import * as datalineage from "@distilled.cloud/gcp/datalineage_v1";
import { Unowned } from "alchemy/AdoptPolicy";
import { isResolved } from "alchemy/Diff";
import * as Provider from "alchemy/Provider";
import { Resource } from "alchemy/Resource";
import * as Data from "effect/Data";
import * as Effect from "effect/Effect";
import { GcpEnvironment } from "../Environment.ts";
import type { Providers } from "../Providers.ts";
import {
fingerprint,
hasAlchemyAttributeMap,
listLineageEvents,
listOwnedProcesses,
listRuns,
normalizeLocation,
parseName,
processOf,
replaceOnIdentity,
toPhysicalId,
waitUntilGone,
} from "./internal.ts";
export type LineageEntityReference = {
/**
* Fully qualified name of the entity, for example
* `bigquery:project.dataset.table` or `custom:project.system.table`.
*/
fullyQualifiedName: string;
/**
* Optional nested field path within the entity. Empty for asset-level
* lineage.
*/
field?: string[];
};
export type LineageEventLink = {
/** Source entity. */
source: LineageEntityReference;
/** Target entity. */
target: LineageEntityReference;
/**
* How the target depends on the source (`EXACT_COPY` or `OTHER`).
*/
dependencyType?:
| datalineage.GoogleCloudDatacatalogLineageV1DependencyInfoDependencyTypeEnum
| (string & {});
};
export type ProcessesRunsLineageEventProps = {
/**
* Parent run. Full name
* `projects/{project}/locations/{location}/processes/{process}/runs/{run}`
* or a run id combined with `process`. Immutable — changing it replaces
* the event.
*/
run: string;
/**
* Parent process used when `run` is a bare id. Full process name or
* process id combined with `location`.
*/
process?: string;
/**
* Event id (the `{lineage_event}` segment). If omitted, a unique name
* is generated. Max 200 characters: letters, digits, underscores,
* hyphens, colons, periods. Immutable — changing it replaces the event.
*/
lineageEventId?: string;
/**
* Region used when `run` / `process` is a bare id.
* @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
*/
location?: string;
/**
* RFC3339 start of the transformation. Required by the API; Alchemy
* reuses the observed value on subsequent reconciles when omitted.
*/
startTime?: string;
/**
* RFC3339 end of the transformation.
*/
endTime?: string;
/**
* Source-target links (max 100). Lineage events have no labels or
* attributes field, so Alchemy lists events under alchemy-owned
* parent processes for nuke.
*/
links?: LineageEventLink[];
};
export type ProcessesRunsLineageEvent = Resource<
"GCP.DataLineage.ProcessesRunsLineageEvent",
ProcessesRunsLineageEventProps,
{
/** Full resource name `.../runs/{run}/lineageEvents/{lineage_event}`. */
name: string;
/** Lineage event id (last path segment). */
lineageEventId: string;
/** Parent run resource name. */
run: string;
/** Parent process resource name. */
process: string;
/** Project id. */
project: string;
/** Location id. */
location: string;
/** RFC3339 start time. */
startTime: string | undefined;
/** RFC3339 end time. */
endTime: string | undefined;
/** Source-target links. */
links: LineageEventLink[];
},
never,
Providers
>;
/**
* A Data Lineage event — one operation that moved data from source
* assets to target assets.
*
* Events are immutable after create (no patch API). Identity is the
* parent run plus `lineageEventId`. Changing links or timestamps
* replaces the event. Events have no labels, so `list` / nuke walks
* alchemy-owned parent processes.
*
* ### Creating a Lineage Event
* **Example:** One source-to-target link
* ```typescript
* const event = yield* GCP.DataLineage.ProcessesRunsLineageEvent("Load", {
* run: run.name,
* startTime: "2024-01-01T00:00:00Z",
* links: [
* {
* source: { fullyQualifiedName: "custom:proj.raw.orders" },
* target: { fullyQualifiedName: "custom:proj.dw.orders" },
* },
* ],
* });
* ```
*
* @resource
* @category DataLineage
*/
export const ProcessesRunsLineageEvent = Resource<ProcessesRunsLineageEvent>(
"GCP.DataLineage.ProcessesRunsLineageEvent",
);
/** Alias matching the Data Lineage resource name. */
export const LineageEvent = ProcessesRunsLineageEvent;
export type LineageEvent = ProcessesRunsLineageEvent;
export class ProcessesRunsLineageEventNotResolved extends Data.TaggedError(
"GCP.DataLineage.ProcessesRunsLineageEventNotResolved",
)<{
resourceName: string;
}> {}
const resourceName = (run: string, lineageEventId: string) =>
`${run}/lineageEvents/${lineageEventId}`;
const nowIso = () => Effect.sync(() => new Date().toISOString());
const runOf = (run: string, process: string | undefined, project: string, location: string) => {
if (run.includes("/")) return run.replace(/\/+$/, "");
const parent = processOf(process ?? "", project, location);
return `${parent}/runs/${run}`;
};
const linksOf = (
links: readonly LineageEventLink[] | undefined,
): datalineage.GoogleCloudDatacatalogLineageV1EventLink[] =>
(links ?? []).map((link) => ({
source: {
fullyQualifiedName: link.source.fullyQualifiedName,
field: link.source.field,
},
target: {
fullyQualifiedName: link.target.fullyQualifiedName,
field: link.target.field,
},
dependencyInfo:
link.dependencyType !== undefined ? { dependencyType: link.dependencyType } : undefined,
}));
const linksFrom = (
links: readonly datalineage.GoogleCloudDatacatalogLineageV1EventLink[] | undefined,
): LineageEventLink[] =>
(links ?? [])
.filter(
(link) =>
(link.source?.fullyQualifiedName ?? "").length > 0 &&
(link.target?.fullyQualifiedName ?? "").length > 0,
)
.map((link) => ({
source: {
fullyQualifiedName: link.source?.fullyQualifiedName ?? "",
field: link.source?.field,
},
target: {
fullyQualifiedName: link.target?.fullyQualifiedName ?? "",
field: link.target?.field,
},
dependencyType: link.dependencyInfo?.dependencyType,
}));
const toAttrs = (
event: datalineage.GoogleCloudDatacatalogLineageV1LineageEvent,
project: string,
region: string,
) => {
const name = event.name ?? "";
const parsed = parseName(name, "lineageEvents", region);
const process = parseName(parsed.parent, "runs", region).parent;
return {
name,
lineageEventId: parsed.id,
run: parsed.parent,
process,
project: parsed.project || project,
location: parsed.location,
startTime: event.startTime,
endTime: event.endTime,
links: linksFrom(event.links),
};
};
const getByName = (name: string) =>
name.length === 0
? Effect.undefined
: datalineage
.getProjectsLocationsProcessesRunsLineageEvents({ name })
.pipe(Effect.catchTag("NotFound", () => Effect.undefined));
const getProcess = (name: string) =>
name.length === 0
? Effect.undefined
: datalineage
.getProjectsLocationsProcesses({ name })
.pipe(Effect.catchTag("NotFound", () => Effect.undefined));
const identityFingerprint = (news: {
startTime?: string;
endTime?: string;
links?: LineageEventLink[];
}) =>
fingerprint({
startTime: news.startTime,
endTime: news.endTime,
links: linksOf(news.links),
});
export const ProcessesRunsLineageEventProvider = () =>
Provider.succeed(ProcessesRunsLineageEvent, {
stables: ["name", "lineageEventId", "run", "process", "project", "location"],
diff: Effect.fn(function* ({ news, olds, output }) {
if (!isResolved(news)) return undefined;
const env = yield* GcpEnvironment.current;
const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
const nextLocation = normalizeLocation(
news.location ?? olds?.location ?? output?.location,
env.region,
);
const previous = identityFingerprint({
startTime: olds?.startTime ?? output?.startTime,
endTime: olds?.endTime ?? output?.endTime,
links: olds?.links ?? output?.links,
});
const next = identityFingerprint({
startTime: news.startTime ?? olds?.startTime ?? output?.startTime,
endTime: news.endTime ?? olds?.endTime ?? output?.endTime,
links: news.links ?? olds?.links ?? output?.links,
});
return replaceOnIdentity({
previousId: olds?.lineageEventId ?? output?.lineageEventId,
nextId: news.lineageEventId,
previousParent: olds?.run ?? output?.run,
nextParent: news.run,
extra:
(output !== undefined && previousLocation !== nextLocation) ||
(output !== undefined && previous !== next),
});
}),
read: Effect.fn(function* ({ id, olds, output }) {
const env = yield* GcpEnvironment.current;
const parentRef = olds?.run ?? output?.run;
// A recovery read of an interrupted create gets persisted props whose
// unresolved parent reference was stripped; without the parent there
// is no name to look up, so report nothing found rather than calling
// the API with an empty path segment.
if (output?.name === undefined && (typeof parentRef !== "string" || parentRef.length === 0)) {
return undefined;
}
const location = normalizeLocation(olds?.location ?? output?.location, env.region);
const run = runOf(
olds?.run ?? output?.run ?? "",
olds?.process ?? output?.process,
env.project,
location,
);
const lineageEventId = yield* toPhysicalId(id, olds?.lineageEventId, output?.lineageEventId);
const name = output?.name ?? resourceName(run, lineageEventId);
const existing = yield* getByName(name);
if (existing === undefined) return undefined;
const attrs = toAttrs(existing, env.project, env.region);
const parent = yield* getProcess(attrs.process);
return parent === undefined || hasAlchemyAttributeMap(parent.attributes)
? attrs
: Unowned(attrs);
}),
list: () =>
Effect.gen(function* () {
const env = yield* GcpEnvironment.current;
const processes = yield* listOwnedProcesses(env.project, env.region);
const runs = (yield* Effect.forEach(processes, (process) => listRuns(process.name ?? ""), {
concurrency: 4,
})).flat();
const events = (yield* Effect.forEach(runs, (run) => listLineageEvents(run.name ?? ""), {
concurrency: 4,
})).flat();
return events.map((event) => toAttrs(event, env.project, env.region));
}),
reconcile: Effect.fn(function* ({ id, news, output }) {
const env = yield* GcpEnvironment.current;
const location = normalizeLocation(news.location ?? output?.location, env.region);
const run = runOf(news.run, news.process, env.project, location);
const lineageEventId = yield* toPhysicalId(id, news.lineageEventId, output?.lineageEventId);
const name = output?.name ?? resourceName(run, lineageEventId);
let current = yield* getByName(name);
if (current === undefined) {
const startTime = news.startTime ?? (yield* nowIso());
const created = yield* datalineage
.createProjectsLocationsProcessesRunsLineageEvents({
parent: run,
body: {
name,
startTime,
endTime: news.endTime,
links: linksOf(news.links),
},
})
.pipe(Effect.catchTag("Conflict", () => getByName(name)));
current = created ?? undefined;
}
if (current === undefined) {
return yield* new ProcessesRunsLineageEventNotResolved({ resourceName: name });
}
return toAttrs(current, env.project, env.region);
}),
delete: Effect.fn(function* ({ output }) {
if (!output.name) return;
yield* datalineage
.deleteProjectsLocationsProcessesRunsLineageEvents({
name: output.name,
allowMissing: true,
})
.pipe(Effect.catchTag("NotFound", () => Effect.void));
yield* waitUntilGone(getByName(output.name), output.name);
}),
});