sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ AWS/ Lambda/ Telemetry.test.ts

import * as AWS from "@alchemy.run/aws";
import * as Cloudflare from "@alchemy.run/cloudflare";
import { describe, expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
import * as Clock from "effect/Clock";
import * as ConfigProvider from "effect/ConfigProvider";
import * as Effect from "effect/Effect";
import * as HttpClient from "effect/http/HttpClient";
import * as Layer from "effect/Layer";
import * as Schedule from "effect/Schedule";
import * as pathe from "pathe";
import { expectUrlContains } from "../../Cloudflare/Utils/Http.ts";
import type { OtelSink } from "./fixtures/otel-collector-worker.ts";
import { OtelTestFunction, OtelTestFunctionLive } from "./fixtures/otel-handler.ts";

const { test } = Test.make({ providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()) });

const collectorMain = pathe.resolve(import.meta.dirname, "fixtures/otel-collector-worker.ts");

// The OTLP sink the Lambda exports to. A workers.dev URL is fine here —
// the 1042 worker-to-worker restriction doesn't apply to requests coming
// from Lambda. Declared identically in both deploy steps so the second
// deploy keeps it.
const collectorWorker = () =>
  Cloudflare.Worker("OtelCollector", {
    main: collectorMain,
    env: { SINK: Cloudflare.DurableObject<OtelSink>("OtelSink") },
    compatibility: { date: "2024-09-23" },
  });

interface CollectedSpan {
  name: string;
  kind?: number;
  status?: { code?: number; message?: string };
  attributes?: { key: string; value: Record<string, unknown> }[];
}

// Every span across all the OTLP traces pushes the collector recorded.
const collectedSpans = (collected: unknown): CollectedSpan[] =>
  (collected as { items: { signal: string; payload: any }[] }).items
    .filter((item) => item.signal === "traces")
    .flatMap((item) => item.payload.resourceSpans ?? [])
    .flatMap((resource: any) => resource.scopeSpans ?? [])
    .flatMap((scope: any) => (scope.spans ?? []) as CollectedSpan[]);

// Name, path and status of each span, printed when an assertion fails.
const rootSpanSummary = (collected: unknown) =>
  collectedSpans(collected).map((span) => ({
    name: span.name,
    path: span.attributes?.find((attribute) => attribute.key === "url.path")?.value,
    status: span.status?.code,
  }));

// OTLP SPAN_KIND_SERVER: the HttpMiddleware tracer's root span per request.
const SERVER_SPAN_KIND = 2;

// The HTTP server root spans for one path.
const rootSpansFor = (collected: unknown, path: string): CollectedSpan[] =>
  collectedSpans(collected).filter(
    (span) =>
      span.kind === SERVER_SPAN_KIND &&
      span.attributes?.some(
        (attribute) => attribute.key === "url.path" && attribute.value.stringValue === path,
      ),
  );

describe(
  "AWS.Lambda Telemetry",
  {
    tags: [
      "provider:aws",
      "provider:aws:lambda",
      "provider:cloudflare",
      "provider:cloudflare:worker",
      "live",
    ],
  },
  () => {
    test.provider(
      "Lambda exports OTLP telemetry per invocation",
      (stack) =>
        Effect.gen(function* () {
          yield* stack.destroy();

          // Deploy the collector first: the Lambda's OTLP endpoint is
          // resolved from the deployer's environment at deploy time.
          const collector = yield* stack.deploy(
            Effect.gen(function* () {
              return yield* collectorWorker();
            }),
          );
          const collected = `${collector.url}/collected`;
          yield* expectUrlContains(collected, "otel-collector-ok", { timeout: "180 seconds" });

          // The fixture's telemetry binding layer reads `COLLECTOR_URL` via
          // `Config` at Init. The test harness snapshots its ConfigProvider
          // before the test body runs, so layer the just-learned collector
          // URL on top of the current provider for this deploy only.
          const currentConfig = yield* ConfigProvider.ConfigProvider;
          const { fn } = yield* stack
            .deploy(
              Effect.gen(function* () {
                yield* collectorWorker();
                const fn = yield* OtelTestFunction.pipe(Effect.provide(OtelTestFunctionLive));
                return { fn };
              }),
            )
            .pipe(
              Effect.provideService(
                ConfigProvider.ConfigProvider,
                ConfigProvider.orElse(
                  ConfigProvider.fromUnknown({ COLLECTOR_URL: collector.url }),
                  currentConfig,
                ),
              ),
            );

          // Wait for the function URL to serve AND for the collector to be
          // reachable from the Lambda's region — workers.dev propagates
          // per-PoP, so the Lambda can see placeholder 404s long after the
          // test machine gets 200s.
          const fnUrl = (fn.functionUrl as string).replace(/\/$/, "");
          yield* expectUrlContains(`${fnUrl}/probe`, '"status":200', {
            timeout: "240 seconds",
            label: "collector reachable from Lambda",
          });

          // Drive one traced invocation.
          yield* expectUrlContains(`${fnUrl}/work`, "lambda-did-work", { timeout: "120 seconds" });

          // The invocation scope's flush ships everything the invocation
          // produced before the response is returned.
          yield* expectUrlContains(collected, "otel-lambda-test", {
            timeout: "120 seconds",
            label: "lambda service.name",
          });
          // HTTP server root span from the HttpMiddleware tracer, named
          // after the request method.
          yield* expectUrlContains(collected, '"name":"GET"');
          // Child span from Effect.fn instrumentation.
          yield* expectUrlContains(collected, "lambda.child-span");
          // Log record shipped by the OTLP logger.
          yield* expectUrlContains(collected, "lambda-work-log");

          // An invocation that runs past its 3 s timeout is interrupted:
          // its finalizer runs, the span of the call that hung ends, and the
          // root span ends with the timeout — all exported. The invocation
          // is then held until Lambda's own 5 s timeout ends it, so the
          // caller sees Lambda's timeout error, not a response from the
          // handler.
          const client = yield* HttpClient.HttpClient;
          const slowStart = yield* Clock.currentTimeMillis;
          const slow = yield* client.get(`${fnUrl}/slow`);
          const slowElapsed = (yield* Clock.currentTimeMillis) - slowStart;
          expect(slow.status).not.toBe(200);
          expect(slowElapsed).toBeGreaterThanOrEqual(4500);

          // Lambda counts it as a real timeout and shuts the sandbox down
          // (our shutdown extension logs the SIGTERM inside the invocation),
          // so work that ignored the interruption cannot resume later.
          const logGroupName = `/aws/lambda/${fn.functionName}`;
          const slowRequestLines = yield* logs.filterLogEvents({ logGroupName }).pipe(
            Effect.map((result) => (result.events ?? []).map((event) => event.message ?? "")),
            Effect.map((lines) => {
              const requestId = lines
                .map((line) => /Lambda invocation (\S+) of .* timed out after/.exec(line)?.[1])
                .find((id) => id !== undefined);
              return requestId === undefined
                ? []
                : lines.filter((line) => line.includes(requestId));
            }),
            Effect.repeat({
              schedule: Schedule.spaced("2 seconds"),
              until: (lines) => lines.some((line) => line.startsWith("REPORT")),
              times: 30,
            }),
          );
          expect(slowRequestLines.find((line) => line.startsWith("REPORT"))).toContain(
            "Status: timeout",
          );
          expect(slowRequestLines.some((line) => line.includes("SIGTERM"))).toBe(true);
          yield* expectUrlContains(collected, "lambda-slow-finalizer", {
            timeout: "120 seconds",
            label: "interrupted invocation's finalizer log",
          });
          yield* expectUrlContains(collected, "lambda-slow-log");
          const slowBody = yield* client
            .get(collected)
            .pipe(Effect.flatMap((response) => response.json));
          const slowRoots = rootSpansFor(slowBody, "/slow");
          expect({ slowRoots: slowRoots.length, roots: rootSpanSummary(slowBody) }).toMatchObject({
            slowRoots: 1,
          });
          expect(slowRoots[0]?.status).toMatchObject({
            code: 2,
            message: expect.stringContaining("timed out"),
          });
          // The span of the call that hung, ended by the interruption.
          const hung = collectedSpans(slowBody).filter((span) => span.name === "lambda.slow-span");
          expect(hung).toHaveLength(1);
          expect(hung[0]?.attributes).toContainEqual({
            key: "effect.fiber.interrupted",
            value: { boolValue: true },
          });

          // An invocation that finishes just inside its timeout is never
          // interrupted, and its root span is a plain success.
          const late = yield* client.get(`${fnUrl}/late`);
          expect(late.status).toBe(200);
          expect(yield* late.text).toContain("lambda-late-done");
          const lateBody = yield* client
            .get(collected)
            .pipe(Effect.flatMap((response) => response.json));
          const lateSpans = rootSpansFor(lateBody, "/late");
          expect({ lateSpans: lateSpans.length, roots: rootSpanSummary(lateBody) }).toMatchObject({
            lateSpans: 1,
          });
          expect(lateSpans[0]?.status?.code).not.toBe(2);

          // An invocation whose finalizer hangs past Lambda's own timeout:
          // Lambda kills it, but 500 ms before that the backstop flush ended
          // the root span and drained the exporters.
          const stuck = yield* client.get(`${fnUrl}/stuck`);
          expect(stuck.status).not.toBe(200);
          const stuckSpans = yield* client.get(collected).pipe(
            Effect.flatMap((response) => response.json),
            Effect.map((body) => rootSpansFor(body, "/stuck")),
            Effect.repeat({
              schedule: Schedule.spaced("2 seconds"),
              until: (spans) => spans.length > 0,
              times: 30,
            }),
          );
          expect(stuckSpans).toHaveLength(1);
          expect(stuckSpans[0]?.status?.code).toBe(2);
          expect(stuckSpans[0]?.attributes).toContainEqual({
            key: "aws.lambda.timeout.imminent",
            value: { boolValue: true },
          });
        }),
      { timeout: 600_000 },
    );
  },
);