import * as ConfigProvider from "effect/ConfigProvider";
import * as Context from "effect/Context";
import type * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Formatter from "effect/Formatter";
import * as HttpClient from "effect/http/HttpClient";
import * as HttpClientRequest from "effect/http/HttpClientRequest";
import * as Layer from "effect/Layer";
import * as OtlpLogger from "effect/observability/OtlpLogger";
import * as OtlpMetrics from "effect/observability/OtlpMetrics";
import * as OtlpSerialization from "effect/observability/OtlpSerialization";
import * as OtlpTracer from "effect/observability/OtlpTracer";
import * as Predicate from "effect/Predicate";
import * as Redacted from "effect/Redacted";
import * as Result from "effect/Result";
import type * as Scope from "effect/Scope";
import { unpackEnvValue } from "./RuntimeContext.ts";
/**
* The shape of the {@link Telemetry} reference: a Layer of telemetry
* exporters (tracer, loggers, metrics) built once per event into the
* event's request scope. Requirements are satisfied from the runtime's
* isolate context (`HttpClient`, `ConfigProvider`, …).
*/
export type TelemetryLayer = Layer.Layer<never, any, any>;
/**
* Read one bound value back at runtime. `rc.set` packs values for the env
* var wire (plain values JSON-stringified, `Redacted` as a marker routed
* through the secret channel), so the raw env string must be unpacked with
* {@link unpackEnvValue} — it handles all three shapes (packed JSON,
* Redacted marker, and a raw string set directly in the environment).
*/
const readBoundValue = (key: string): Effect.Effect<unknown> =>
Effect.flatMap(ConfigProvider.ConfigProvider, (provider) => provider.load([key])).pipe(
// Missing optional bindings are normal on every event. Read their scalar
// directly instead of constructing a Config/Schema error for each absence.
Effect.orDie,
Effect.catchDefect((defect) =>
Predicate.isTagged(defect, "SourceError") ? Effect.undefined : Effect.die(defect),
),
Effect.map((node) => {
const raw = node?.value;
if (raw === undefined || raw === "") {
return undefined;
}
const value = unpackEnvValue<unknown>(raw);
return Redacted.isRedacted(value) ? Redacted.value(value) : value;
}),
);
const readBound = (key: string): Effect.Effect<string | undefined> =>
readBoundValue(key).pipe(
Effect.map((inner) =>
inner === undefined ? undefined : typeof inner === "string" ? inner : Formatter.format(inner),
),
);
/**
* `service.name` fallback chain for the default exporter: the standard
* OTEL variable, then the physical Function/Worker name, then the stack
* name. `OtlpResource.fromConfig` dies without a service name, so the
* chain must always produce one.
*/
const defaultServiceName = Effect.gen(function* () {
return (
(yield* readBound("OTEL_SERVICE_NAME")) ??
(yield* readBound("ALCHEMY_WORKER_NAME")) ??
(yield* readBound("AWS_LAMBDA_FUNCTION_NAME")) ??
(yield* readBound("ALCHEMY_STACK_NAME")) ??
"alchemy"
);
});
const defaultResource = Effect.gen(function* () {
const serviceName = yield* defaultServiceName;
const stack = yield* readBound("ALCHEMY_STACK_NAME");
const stage = yield* readBound("ALCHEMY_STAGE");
return {
serviceName,
attributes: {
...(stack !== undefined ? { "alchemy.stack": stack } : undefined),
...(stage !== undefined ? { "alchemy.stage": stage } : undefined),
},
};
});
/**
* Parse the OpenTelemetry `OTEL_EXPORTER_OTLP_HEADERS` format:
* `key1=value1,key2=value2` with URL-encoded values.
*/
const parseOtlpHeaders = (raw: string | undefined): Record<string, string> | undefined => {
if (raw === undefined || raw === "") {
return undefined;
}
const headers: Record<string, string> = {};
for (const entry of raw.split(",")) {
const eq = entry.indexOf("=");
if (eq === -1) {
continue;
}
const key = entry.slice(0, eq).trim();
if (key === "") {
continue;
}
const value = entry.slice(eq + 1).trim();
try {
headers[key] = decodeURIComponent(value);
} catch {
headers[key] = value;
}
}
return headers;
};
/**
* One export target for one signal, as stored in the bound destination
* list: a concrete OTLP/HTTP URL and the headers to send with it.
*/
export interface ResolvedSignal {
url: string;
headers?: Record<string, string> | undefined;
}
/**
* One export destination in the bound list — the resolved form of one
* {@link layerOtlp} layer.
*/
export interface ResolvedDestination {
traces?: ResolvedSignal | undefined;
logs?: ResolvedSignal | undefined;
metrics?: ResolvedSignal | undefined;
}
/**
* The single env binding carrying every configured destination as a JSON
* array of {@link ResolvedDestination}. One key (instead of per-signal
* keys) is what lets multiple `layerOtlp` layers compose — each layer
* build appends its destination and rebinds the full list.
*/
export const EXPORTERS_KEY = "ALCHEMY_OTEL_EXPORTERS";
/**
* Resolve one signal's endpoint + headers from the standard OpenTelemetry
* env vars (`OTEL_EXPORTER_OTLP_{SIGNAL}_ENDPOINT` as-is, or the base
* endpoint with `/v1/{signal}` appended; headers per-signal → base). This
* forms an *implicit extra destination*, so platform-injected OTLP config
* exports without any layer.
*/
const signalConfig = (signal: "TRACES" | "LOGS" | "METRICS", base: string | undefined) =>
Effect.gen(function* () {
const specific = yield* readBound(`OTEL_EXPORTER_OTLP_${signal}_ENDPOINT`);
const url =
specific !== undefined && specific !== ""
? specific
: base !== undefined && base !== ""
? `${base.replace(/\/$/, "")}/v1/${signal.toLowerCase()}`
: undefined;
if (url === undefined) {
return undefined;
}
const rawHeaders =
(yield* readBound(`OTEL_EXPORTER_OTLP_${signal}_HEADERS`)) ??
(yield* readBound("OTEL_EXPORTER_OTLP_HEADERS"));
return { url, headers: parseOtlpHeaders(rawHeaders) } as ResolvedSignal;
});
/**
* Sentinel URLs the single exporter set posts to; the fanout client
* rewrites each POST to every configured destination for that signal.
* Serializing once through one exporter per signal is what keeps span ids
* consistent across destinations — Effect has a single `Tracer` service,
* so per-destination tracers would generate divergent trace ids.
*/
const SENTINEL = {
traces: "http://telemetry.alchemy.internal/v1/traces",
logs: "http://telemetry.alchemy.internal/v1/logs",
metrics: "http://telemetry.alchemy.internal/v1/metrics",
} as const;
/**
* Wrap the ambient `HttpClient` so a POST to a sentinel URL is re-sent to
* every destination configured for that signal (with the destination's own
* headers). A single healthy destination keeps the exporter alive: the
* first 2xx response is returned; per-destination failures are logged at
* debug and only surface when every destination fails.
*/
const fanoutClient = (
routes: ReadonlyMap<string, ResolvedSignal[]>,
): Layer.Layer<HttpClient.HttpClient, never, HttpClient.HttpClient> =>
Layer.effect(
HttpClient.HttpClient,
Effect.gen(function* () {
const base = yield* HttpClient.HttpClient;
return HttpClient.transform(base, (effect, request) => {
const targets = routes.get(request.url);
if (targets === undefined) {
return effect;
}
return Effect.gen(function* () {
const results = yield* Effect.forEach(
targets,
(target) =>
Effect.result(
base.execute(
request.pipe(
HttpClientRequest.setUrl(target.url),
HttpClientRequest.setHeaders(target.headers ?? {}),
),
),
),
{ concurrency: "unbounded" },
);
const healthy = results.find(
(result) =>
Result.isSuccess(result) &&
result.success.status >= 200 &&
result.success.status < 300,
);
const anySuccess = healthy ?? results.find((result) => Result.isSuccess(result));
if (anySuccess !== undefined && Result.isSuccess(anySuccess)) {
for (const result of results) {
if (result !== anySuccess && Result.isFailure(result)) {
yield* Effect.logDebug("telemetry destination failed", result.failure);
}
}
return anySuccess.success;
}
const failure = results.find((result) => Result.isFailure(result));
return yield* failure !== undefined && Result.isFailure(failure)
? Effect.fail(failure.failure)
: effect;
});
});
}),
);
/**
* Every destination this runtime exports to: the bound `layerOtlp` list,
* plus the standard `OTEL_EXPORTER_OTLP_*` variables as an implicit extra
* destination. Resolved per call, since providers can change between events.
*
* @internal
*/
export const configuredDestinations: Effect.Effect<ResolvedDestination[]> = Effect.gen(
function* () {
// Depending on the secret/plain packing path, the bound list arrives
// as a JSON string or already parsed into an array.
const rawList = yield* readBoundValue(EXPORTERS_KEY);
const bound: ResolvedDestination[] = Array.isArray(rawList)
? (rawList as ResolvedDestination[])
: typeof rawList === "string" && rawList !== ""
? yield* Effect.try(() => JSON.parse(rawList) as ResolvedDestination[]).pipe(Effect.orDie)
: [];
const baseEndpoint = yield* readBound("OTEL_EXPORTER_OTLP_ENDPOINT");
const [stdTraces, stdLogs, stdMetrics] = yield* Effect.all([
signalConfig("TRACES", baseEndpoint),
signalConfig("LOGS", baseEndpoint),
signalConfig("METRICS", baseEndpoint),
]);
return [
...bound,
...(stdTraces || stdLogs || stdMetrics
? [{ traces: stdTraces, logs: stdLogs, metrics: stdMetrics }]
: []),
];
},
);
const makeExporterLayer = (options?: {
exportInterval?: Duration.Input;
shutdownTimeout?: Duration.Input;
}): TelemetryLayer =>
Layer.unwrap(
Effect.gen(function* () {
const destinations = yield* configuredDestinations;
const targets = (signal: keyof ResolvedDestination) =>
destinations.flatMap((destination) => {
const resolved = destination[signal];
return resolved !== undefined ? [resolved] : [];
});
const traces = targets("traces");
const logs = targets("logs");
const metrics = targets("metrics");
if (traces.length === 0 && logs.length === 0 && metrics.length === 0) {
return Layer.empty;
}
const resource = yield* defaultResource;
const routes = new Map<string, ResolvedSignal[]>();
const layers: Layer.Layer<never, never, any>[] = [];
if (traces.length > 0) {
routes.set(SENTINEL.traces, traces);
layers.push(
OtlpTracer.layer({
url: SENTINEL.traces,
resource,
exportInterval: options?.exportInterval,
shutdownTimeout: options?.shutdownTimeout,
}),
);
}
if (logs.length > 0) {
routes.set(SENTINEL.logs, logs);
layers.push(
OtlpLogger.layer({
url: SENTINEL.logs,
resource,
exportInterval: options?.exportInterval,
shutdownTimeout: options?.shutdownTimeout,
}),
);
}
if (metrics.length > 0) {
routes.set(SENTINEL.metrics, metrics);
layers.push(
OtlpMetrics.layer({
url: SENTINEL.metrics,
resource,
exportInterval: options?.exportInterval,
shutdownTimeout: options?.shutdownTimeout,
}).pipe(
// Metrics go out as protobuf: OTLP/HTTP receivers must accept
// it, while JSON is optional and some reject it on /v1/metrics
// (Axiom answers 415 Unsupported Media Type).
Layer.provide(OtlpSerialization.layerProtobuf),
),
);
}
return Layer.mergeAll(...(layers as [Layer.Layer<never>])).pipe(
Layer.provide(OtlpSerialization.layerJson),
Layer.provide(fanoutClient(routes)),
);
}).pipe(
Effect.catchCause((cause) =>
Effect.logWarning("Invalid telemetry configuration; telemetry disabled", cause).pipe(
Effect.as(Layer.empty),
),
),
),
);
/**
* The runtime half of the {@link layerOtlp} binding, and the default
* per-event Layer: reads the bound `OTEL_EXPORTER_OTLP_*` values back and
* constructs the OTLP exporters (JSON for traces and logs, protobuf for
* metrics). Each signal resolves independently;
* only configured signals export; resolves to `Layer.empty` when nothing is
* bound, so no exporters are built until a destination is configured.
*
* The periodic export intervals are effectively disabled: the exporter is
* built per event and the request-scope flush delivers everything. An
* interval export firing mid-event would race the scope close — the close
* interrupts the exporter's in-flight batch (already spliced out of the
* buffer), silently dropping it. Lambda invocations regularly outlive the
* 1-second logger interval, which is exactly how this was discovered.
* Final exports get the OTLP ten-second batch budget rather than the
* exporter's three-second shutdown default, which can cancel slow delivery.
*
* A malformed configuration degrades to `Layer.empty` with a warning
* instead of failing the event.
*/
export const fromBoundConfig: TelemetryLayer = makeExporterLayer({
exportInterval: "1 hour",
shutdownTimeout: "10 seconds",
});
/**
* The process-runtime half of the binding: same bound-value resolution with
* the standard periodic export intervals, since a server's root scope only
* flushes at shutdown.
*/
const fromBoundConfigProcess: TelemetryLayer = makeExporterLayer();
/**
* The per-event telemetry exporters, as a `Context.Reference` holding the
* Layer the runtime bridges build into every event's request scope.
* Provide it via {@link layerOtlp} / {@link layer} rather than directly —
* see the module documentation.
*/
export const Telemetry = Context.Reference<TelemetryLayer>("alchemy/Telemetry", {
defaultValue: () => fromBoundConfig,
});
const reference = Telemetry;
/**
* Build the configured {@link Telemetry} Layer into an event's request
* scope, returning the Context of telemetry services to provide to the
* event's handler effect.
*
* Called by the runtime bridges (Worker, Durable Object, Workflow, Lambda)
* once per event. Building into the *request* scope — not the
* never-finalized isolate scope — is what makes export work on workerd:
* the batching fiber lives inside the event's I/O context and the final
* flush runs from the scope's finalizer, which the bridges register with
* `ctx.waitUntil`.
*
* A failed build (bad user Layer, config error) degrades to an empty
* Context with a warning instead of failing the event.
*
* `override` is the (possibly merged) custom Layer registered on the
* runtime context by {@link layer} during init. It composes with
* — rather than replaces — the bound OTLP destinations: loggers and metric
* exporters merge, and a custom `Tracer` (a single Effect service) wins
* over the built-in one.
*
* Declared `R = never`: the Layer's actual requirements (`HttpClient`,
* `ConfigProvider`, …) are satisfied at runtime by the bridge's surrounding
* `Effect.provide` of the built runtime context.
*/
export const buildEventTelemetry = (
context: Context.Context<never>,
scope: Scope.Scope,
override?: TelemetryLayer,
base?: TelemetryLayer,
): Effect.Effect<Context.Context<never>> =>
Effect.suspend(() => {
const bound = base ?? Context.get(context, reference);
return Layer.buildWithScope(
override === undefined ? bound : Layer.mergeAll(bound, override),
scope,
);
}).pipe(
Effect.catchCause((cause) =>
Effect.logWarning("Failed to build telemetry layer", cause).pipe(Effect.as(Context.empty())),
),
) as Effect.Effect<Context.Context<never>>;
/**
* Provide telemetry to a long-running server process (Cloudflare Container,
* ECS Task, EC2 host, Lambda microVM).
*
* Unlike the per-event runtime bridges, server processes have no I/O-context
* pinning and a real shutdown: the configured {@link Telemetry} Layer is
* built ONCE into the ambient root scope, exporters batch on their intervals
* for the life of the process, and the final flush runs when the root scope
* closes on graceful exit.
*
* `runtimeContext` is the entrypoint's runtime context; its `telemetry`
* field carries the custom Layer(s) registered by {@link layer}
* during init, composed with the bound OTLP destinations (read with the
* standard periodic export intervals).
*/
export const provideProcessTelemetry =
(runtimeContext?: { telemetry?: TelemetryLayer | undefined }) =>
<A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R | Scope.Scope> =>
Effect.gen(function* () {
const context = yield* Effect.context<never>();
const scope = yield* Effect.scope;
const telemetry = yield* buildEventTelemetry(
context,
scope,
runtimeContext?.telemetry,
fromBoundConfigProcess,
);
return yield* Effect.provideContext(effect, telemetry);
});