sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ AWS/ StateStore/ State.test.ts

import { createHash } from "node:crypto";
import * as AWS from "@alchemy.run/aws";
import { makeS3State } from "@alchemy.run/aws";
import { createStateBucketName } from "@alchemy.run/aws/StateStore/State";
import { describe, expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as Core from "@alchemy.run/test/Core";
import * as kms from "@distilled.cloud/aws/kms";
import * as s3 from "@distilled.cloud/aws/s3";
import * as sqs from "@distilled.cloud/aws/sqs";
import * as Alchemy from "alchemy";
import { StateStoreError, type ResourceState, type StateService } from "alchemy/State";
import * as Effect from "effect/Effect";
import * as Logger from "effect/Logger";
import * as Redacted from "effect/Redacted";
import * as References from "effect/References";
import * as Schedule from "effect/Schedule";
import * as Stream from "effect/Stream";

const { test } = Test.make({ providers: AWS.providers() });

test.provider(
  "PR1587 fresh state services skip matching KMS encryption writes",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const { bucket, first, second } = yield* stack.deploy(
        Effect.gen(function* () {
          const first = yield* AWS.KMS.Key("FirstStateKey", {
            deletionWindow: "7 days",
          });
          const second = yield* AWS.KMS.Key("SecondStateKey", {
            deletionWindow: "7 days",
          });
          const bucket = yield* AWS.S3.Bucket("IdentityStateBucket", {
            encryption: {
              sseAlgorithm: "aws:kms",
              kmsMasterKeyId: first.keyArn,
            },
          });
          return { bucket, first, second };
        }),
      );
      const initialize = (key: string, bucketKeyEnabled?: boolean) =>
        Effect.gen(function* () {
          const state = yield* makeS3State({
            bucketName: bucket.bucketName,
            prefix: "pr1587",
            encryption: {
              sseAlgorithm: "aws:kms",
              kmsMasterKeyId: key,
              bucketKeyEnabled,
            },
          });
          return yield* state.listStacks();
        });
      expect(yield* initialize(first.keyArn)).toEqual([]);
      const observed = yield* s3.getBucketEncryption({
        Bucket: bucket.bucketName,
      });
      const decoded =
        observed.ServerSideEncryptionConfiguration!.Rules[0]!.ApplyServerSideEncryptionByDefault!
          .KMSMasterKeyID;
      expect(Redacted.isRedacted(decoded)).toBe(true);
      expect(Redacted.isRedacted(decoded) ? Redacted.value(decoded) : decoded).toBe(first.keyArn);
      const write = s3.putBucketEncryption({
        Bucket: bucket.bucketName,
        ServerSideEncryptionConfiguration: observed.ServerSideEncryptionConfiguration!,
      });
      const deniedWrite = write.pipe(
        Effect.as(false),
        Effect.catchTag("AccessDeniedException", () => Effect.succeed(true)),
      );
      yield* Effect.gen(function* () {
        yield* s3.putBucketPolicy({
          Bucket: bucket.bucketName,
          Policy: JSON.stringify({
            Version: "2012-10-17",
            Statement: [
              {
                Effect: "Deny",
                Principal: "*",
                Action: "s3:PutEncryptionConfiguration",
                Resource: bucket.bucketArn,
              },
            ],
          }),
        });
        expect(
          yield* deniedWrite.pipe(
            Effect.repeat({
              until: Boolean,
              schedule: Schedule.spaced("1 second"),
              times: 8,
            }),
          ),
        ).toBe(true);
        // Each call constructs a fresh service, bypassing the per-service initialization cache.
        expect(yield* initialize(first.keyArn)).toEqual([]);
        expect(yield* initialize(first.keyArn, false)).toEqual([]);
        expect(yield* deniedWrite).toBe(true);
      }).pipe(
        Effect.ensuring(s3.deleteBucketPolicy({ Bucket: bucket.bucketName }).pipe(Effect.orDie)),
      );
      yield* write.pipe(
        Effect.retry({
          while: (error) => error._tag === "AccessDeniedException",
          schedule: Schedule.spaced("1 second"),
          times: 8,
        }),
      );

      yield* initialize(second.keyArn);
      const changed = (yield* s3.getBucketEncryption({
        Bucket: bucket.bucketName,
      })).ServerSideEncryptionConfiguration!.Rules[0]!.ApplyServerSideEncryptionByDefault!
        .KMSMasterKeyID;
      expect(Redacted.isRedacted(changed) ? Redacted.value(changed) : changed).toBe(second.keyArn);
      yield* initialize(second.keyArn, true);
      expect(
        (yield* s3.getBucketEncryption({ Bucket: bucket.bucketName }))
          .ServerSideEncryptionConfiguration?.Rules[0]?.BucketKeyEnabled,
      ).toBe(true);
      const defaultState = yield* makeS3State({
        bucketName: bucket.bucketName,
        prefix: "pr1587",
      });
      expect(yield* defaultState.listStacks()).toEqual([]);
      const defaults = (yield* s3.getBucketEncryption({
        Bucket: bucket.bucketName,
      })).ServerSideEncryptionConfiguration!.Rules[0]!;
      expect(defaults.ApplyServerSideEncryptionByDefault?.SSEAlgorithm).toBe("AES256");
      expect(defaults.ApplyServerSideEncryptionByDefault?.KMSMasterKeyID).toBeUndefined();
      expect(defaults.BucketKeyEnabled ?? false).toBe(false);
      expect(defaults.BlockedEncryptionTypes?.EncryptionType).toEqual(["NONE"]);
      yield* stack.destroy();
      const absent = yield* s3.getBucketLocation({ Bucket: bucket.bucketName }).pipe(
        Effect.as(false),
        Effect.catchTag("NoSuchBucket", () => Effect.succeed(true)),
        Effect.repeat({
          until: Boolean,
          schedule: Schedule.spaced("1 second"),
          times: 8,
        }),
      );
      expect(absent).toBe(true);
      for (const key of [first, second]) {
        expect((yield* kms.describeKey({ KeyId: key.keyId })).KeyMetadata?.KeyState).toBe(
          "PendingDeletion",
        );
      }
    }),
  {
    tags: [
      "provider:aws",
      "provider:aws:kms",
      "provider:aws:s3",
      "provider:aws:statestore",
      "live",
    ],
    timeout: 120_000,
  },
);

for (const blocked of ["SSE-C", "NONE"] as const) {
  test.provider(
    `state initialization restores encryption defaults after external ${blocked} settings`,
    (stack) =>
      Effect.gen(function* () {
        yield* stack.destroy();
        const bucket = yield* stack.deploy(AWS.S3.Bucket("EncryptionBlocksStateBucket", {}));
        const initialize = Effect.gen(function* () {
          const state = yield* makeS3State({
            bucketName: bucket.bucketName,
            prefix: "pr1588",
          });
          return yield* state.listStacks();
        });
        expect(yield* initialize).toEqual([]);
        yield* s3.putBucketEncryption({
          Bucket: bucket.bucketName,
          ServerSideEncryptionConfiguration: {
            Rules: [
              {
                ApplyServerSideEncryptionByDefault: { SSEAlgorithm: "aws:kms" },
                BlockedEncryptionTypes: { EncryptionType: [blocked] },
              },
            ],
          },
        });
        const before = (yield* s3.getBucketEncryption({
          Bucket: bucket.bucketName,
        })).ServerSideEncryptionConfiguration!.Rules[0]!;
        expect(before.BlockedEncryptionTypes?.EncryptionType).toEqual([blocked]);
        expect(before.ApplyServerSideEncryptionByDefault?.SSEAlgorithm).toBe("aws:kms");
        // Re-running the generator constructs a fresh state service and rechecks cloud configuration.
        expect(yield* initialize).toEqual([]);
        const after = (yield* s3.getBucketEncryption({
          Bucket: bucket.bucketName,
        })).ServerSideEncryptionConfiguration!.Rules[0]!;
        expect(after.ApplyServerSideEncryptionByDefault?.SSEAlgorithm).toBe("AES256");
        expect(after.ApplyServerSideEncryptionByDefault?.KMSMasterKeyID).toBeUndefined();
        expect(after.BucketKeyEnabled ?? false).toBe(false);
        expect(after.BlockedEncryptionTypes?.EncryptionType).toEqual(["NONE"]);
        yield* stack.destroy();
        const absent = yield* s3.getBucketLocation({ Bucket: bucket.bucketName }).pipe(
          Effect.as(false),
          Effect.catchTag("NoSuchBucket", () => Effect.succeed(true)),
          Effect.repeat({
            until: Boolean,
            schedule: Schedule.spaced("1 second"),
            times: 8,
          }),
        );
        expect(absent).toBe(true);
      }),
    {
      tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
      timeout: 120_000,
    },
  );
}

test.provider(
  "state services manage encryption blocks and restore defaults on removal",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const { bucket, blockedBucket } = yield* stack.deploy(
        Effect.gen(function* () {
          const bucket = yield* AWS.S3.Bucket("ManagedStateEncryptionBlocks", {});
          const blockedBucket = yield* AWS.S3.Bucket("BlockedNoOpStateBucket", {});
          return { bucket, blockedBucket };
        }),
      );
      const initialize = (
        blockedEncryptionTypes: AWS.S3.BucketEncryption["blockedEncryptionTypes"],
        bucketName = bucket.bucketName,
      ) =>
        Effect.gen(function* () {
          const state = yield* makeS3State({
            bucketName,
            prefix: "pr1588-managed",
            encryption: { sseAlgorithm: "AES256", blockedEncryptionTypes },
          });
          return yield* state.listStacks();
        });
      const readRule = s3
        .getBucketEncryption({ Bucket: bucket.bucketName })
        .pipe(Effect.map((result) => result.ServerSideEncryptionConfiguration!.Rules[0]!));
      const block = s3.putBucketEncryption({
        Bucket: bucket.bucketName,
        ServerSideEncryptionConfiguration: {
          Rules: [
            {
              ApplyServerSideEncryptionByDefault: { SSEAlgorithm: "AES256" },
              BlockedEncryptionTypes: { EncryptionType: ["SSE-C"] },
            },
          ],
        },
      });
      yield* block;
      expect(yield* initialize([])).toEqual([]);
      expect((yield* readRule).BlockedEncryptionTypes?.EncryptionType).toEqual(["NONE"]);
      yield* block;
      expect(yield* initialize([])).toEqual([]);
      expect((yield* readRule).BlockedEncryptionTypes?.EncryptionType).toEqual(["NONE"]);
      expect(yield* initialize(["SSE-C"])).toEqual([]);
      expect((yield* readRule).BlockedEncryptionTypes?.EncryptionType).toEqual(["SSE-C"]);
      expect(yield* initialize(undefined)).toEqual([]);
      expect((yield* readRule).BlockedEncryptionTypes?.EncryptionType).toEqual(["NONE"]);

      // Keep each deny policy until bucket deletion; policy removal is eventually consistent.
      const settings: "SSE-C"[][] = [[], ["SSE-C"]];
      for (const types of settings) {
        const target = types.length ? blockedBucket : bucket;
        expect(yield* initialize(types, target.bucketName)).toEqual([]);
        const rule = (yield* s3.getBucketEncryption({
          Bucket: target.bucketName,
        })).ServerSideEncryptionConfiguration!.Rules[0]!;
        expect(rule.BlockedEncryptionTypes?.EncryptionType).toEqual(
          types.length ? ["SSE-C"] : ["NONE"],
        );
        const probeWrite = s3
          .putBucketEncryption({
            Bucket: target.bucketName,
            ServerSideEncryptionConfiguration: {
              Rules: [
                {
                  ApplyServerSideEncryptionByDefault: {
                    SSEAlgorithm: "AES256",
                  },
                  BucketKeyEnabled: false,
                  BlockedEncryptionTypes: {
                    EncryptionType: types.length ? types : ["NONE"],
                  },
                },
              ],
            },
          })
          .pipe(
            Effect.as(false),
            Effect.catchTag("AccessDeniedException", () => Effect.succeed(true)),
          );
        yield* s3.putBucketPolicy({
          Bucket: target.bucketName,
          Policy: JSON.stringify({
            Version: "2012-10-17",
            Statement: [
              {
                Effect: "Deny",
                Principal: "*",
                Action: "s3:PutEncryptionConfiguration",
                Resource: target.bucketArn,
              },
            ],
          }),
        });
        expect(
          yield* probeWrite.pipe(
            Effect.repeat({
              until: Boolean,
              schedule: Schedule.spaced("1 second"),
              times: 8,
            }),
          ),
        ).toBe(true);
        expect(yield* initialize([...types, ...types], target.bucketName)).toEqual([]);
        if (!types.length) {
          expect(yield* initialize(undefined, target.bucketName)).toEqual([]);
        }
        expect(yield* probeWrite).toBe(true);
      }
      yield* stack.destroy();
      for (const target of [bucket, blockedBucket]) {
        const absent = yield* s3.getBucketLocation({ Bucket: target.bucketName }).pipe(
          Effect.as(false),
          Effect.catchTag("NoSuchBucket", () => Effect.succeed(true)),
          Effect.repeat({
            until: Boolean,
            schedule: Schedule.spaced("1 second"),
            times: 8,
          }),
        );
        expect(absent).toBe(true);
      }
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

const STACK = "S3StateStoreTestStack";

/**
 * Guaranteed out-of-band cleanup: `deleteStack` is idempotent (list +
 * batched delete), and `Effect.orDie` collapses the finalizer's error
 * channel to `never` so it is a valid `Effect.ensuring` finalizer.
 * Every test pre-cleans AND finalizes with this so a failing assertion
 * can never leave state objects behind in the bucket.
 */
const cleanStage = (state: StateService, stage: string) =>
  state.deleteStack({ stack: STACK, stage }).pipe(Effect.orDie);

const resource = (fqn: string, attr: Record<string, unknown>): ResourceState =>
  ({
    resourceType: "test:resource",
    namespace: undefined,
    fqn,
    logicalId: fqn,
    instanceId: `instance-${fqn}`,
    providerVersion: 1,
    status: "created",
    downstream: [],
    bindings: [],
    props: {},
    attr,
  }) as ResourceState;

test.provider(
  "set/get/list/delete round-trips state through S3",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(AWS.S3.Bucket("StateBucket", { forceDestroy: true }));
      const state = yield* makeS3State({
        bucketName: bucket.bucketName,
        prefix: "test-state",
      });
      const stage = "round-trip";

      // start from a clean slate (idempotent)
      yield* state.deleteStack({ stack: STACK, stage });

      yield* Effect.gen(function* () {
        const a = resource("Parent/ResourceA", { value: "a" });
        const b = resource("ResourceB", { value: "b" });

        yield* state.set({ stack: STACK, stage, fqn: a.fqn, value: a });
        yield* state.set({ stack: STACK, stage, fqn: b.fqn, value: b });

        expect(yield* state.get({ stack: STACK, stage, fqn: a.fqn })).toEqual(a);
        expect(yield* state.get({ stack: STACK, stage, fqn: "does-not-exist" })).toBeUndefined();

        const fqns = yield* state.list({ stack: STACK, stage });
        expect([...fqns].sort()).toEqual(["Parent/ResourceA", "ResourceB"]);

        expect(yield* state.listStacks()).toContain(STACK);
        expect(yield* state.listStages(STACK)).toContain(stage);

        yield* state.delete({ stack: STACK, stage, fqn: a.fqn });
        expect(yield* state.get({ stack: STACK, stage, fqn: a.fqn })).toBeUndefined();
        // deleting a missing resource is a no-op
        yield* state.delete({ stack: STACK, stage, fqn: a.fqn });

        yield* state.deleteStack({ stack: STACK, stage });
        expect(yield* state.list({ stack: STACK, stage })).toEqual([]);
      }).pipe(Effect.ensuring(cleanStage(state, stage)));
      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

// State objects hold generated secrets, and the AWS client logs request
// payloads and parsed responses at Debug. A Debug floor where the store is
// built (the CLI's run log sets one) must not put state contents in the logs.
test.provider(
  "state contents stay out of Debug logs",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(
        AWS.S3.Bucket("LoggingStateBucket", { forceDestroy: true }),
      );
      const secret = "state-logging-secret-0123456789";
      const stage = "logging";

      // Record every log line emitted at a Debug floor.
      const atDebug = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
        Effect.gen(function* () {
          const lines: string[] = [];
          const logger = Logger.make<unknown, void>((options) => {
            lines.push(
              JSON.stringify(options.message, (_, v) => (typeof v === "bigint" ? `${v}` : v)),
            );
          });
          const result = yield* effect.pipe(
            Effect.provide(Logger.layer([logger])),
            Effect.provideService(References.MinimumLogLevel, "Debug"),
          );
          return { result, logs: lines.join("\n") };
        });

      const { logs } = yield* atDebug(
        Effect.gen(function* () {
          const state = yield* makeS3State({ bucketName: bucket.bucketName, prefix: "logging" });
          const value = resource("SecretHolder", { token: secret });
          yield* state.set({ stack: STACK, stage, fqn: value.fqn, value });
          expect(yield* state.get({ stack: STACK, stage, fqn: value.fqn })).toEqual(value);
          yield* state.deleteStack({ stack: STACK, stage });
        }),
      );
      expect(logs).not.toContain(secret);

      // Control: the same payload sent directly is logged at Debug, so the
      // capture above is real.
      const control = yield* atDebug(
        s3.putObject({ Bucket: bucket.bucketName, Key: "control", Body: secret }),
      );
      expect(control.logs).toContain(secret);

      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

test.provider(
  "existing state buckets converge to secure defaults",
  (stack) =>
    Effect.gen(function* () {
      const { accountId, region } = yield* AWS.AWSEnvironment.current;
      const suffix = yield* Effect.sync(() =>
        createHash("sha256").update(stack.stage).digest("hex").slice(0, 8),
      );
      const bucketName = createStateBucketName(`security-${suffix}-${accountId}`, region);
      const deleteBucket = s3.deleteBucket({ Bucket: bucketName }).pipe(
        Effect.catchTag("NoSuchBucket", () => Effect.void),
        Effect.orDie,
      );

      yield* s3
        .deleteBucket({ Bucket: bucketName })
        .pipe(Effect.catchTag("NoSuchBucket", () => Effect.void));

      yield* Effect.gen(function* () {
        // First create the bucket, then simulate out-of-band configuration
        // drift. The second state service targets the SAME existing bucket and
        // must repair every setting from observed cloud state.
        const initial = yield* makeS3State({
          bucketName,
          prefix: "security-test",
        });
        yield* initial.listStacks();

        yield* s3.putBucketVersioning({
          Bucket: bucketName,
          VersioningConfiguration: { Status: "Suspended" },
        });
        yield* s3.putBucketEncryption({
          Bucket: bucketName,
          ServerSideEncryptionConfiguration: {
            Rules: [
              {
                ApplyServerSideEncryptionByDefault: {
                  SSEAlgorithm: "aws:kms",
                },
              },
            ],
          },
        });
        yield* s3.putPublicAccessBlock({
          Bucket: bucketName,
          PublicAccessBlockConfiguration: {
            BlockPublicAcls: false,
            IgnorePublicAcls: false,
            BlockPublicPolicy: false,
            RestrictPublicBuckets: false,
          },
        });
        yield* s3.putBucketOwnershipControls({
          Bucket: bucketName,
          OwnershipControls: {
            Rules: [{ ObjectOwnership: "ObjectWriter" }],
          },
        });

        // Bucket configuration reads are eventually consistent. Observe the
        // injected drift before reconciliation, or a stale AES256 read can
        // incorrectly make the provider conclude that no repair is needed.
        const driftedEncryption = yield* s3.getBucketEncryption({ Bucket: bucketName }).pipe(
          Effect.map(
            (result) =>
              result.ServerSideEncryptionConfiguration?.Rules?.[0]
                ?.ApplyServerSideEncryptionByDefault?.SSEAlgorithm,
          ),
          Effect.repeat({
            until: (algorithm) => algorithm === "aws:kms",
            schedule: Schedule.spaced("1 second"),
            times: 8,
          }),
        );
        expect(driftedEncryption).toBe("aws:kms");

        const secured = yield* makeS3State({
          bucketName,
          prefix: "security-test",
        });
        yield* secured.listStacks();

        const versioning = yield* s3.getBucketVersioning({
          Bucket: bucketName,
        });
        expect(versioning.Status).toBe("Enabled");

        const encryption = yield* s3.getBucketEncryption({
          Bucket: bucketName,
        });
        expect(
          encryption.ServerSideEncryptionConfiguration?.Rules?.[0]
            ?.ApplyServerSideEncryptionByDefault?.SSEAlgorithm,
        ).toBe("AES256");

        const publicAccess = yield* s3.getPublicAccessBlock({
          Bucket: bucketName,
        });
        expect(publicAccess.PublicAccessBlockConfiguration).toEqual({
          BlockPublicAcls: true,
          IgnorePublicAcls: true,
          BlockPublicPolicy: true,
          RestrictPublicBuckets: true,
        });

        const ownership = yield* s3.getBucketOwnershipControls({
          Bucket: bucketName,
        });
        expect(ownership.OwnershipControls?.Rules?.[0]?.ObjectOwnership).toBe(
          "BucketOwnerEnforced",
        );
      }).pipe(Effect.ensuring(deleteBucket));
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

test.provider(
  "stack outputs are stored separately from resources",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(AWS.S3.Bucket("StateBucket", { forceDestroy: true }));
      const state = yield* makeS3State({
        bucketName: bucket.bucketName,
        prefix: "test-state",
      });
      const stage = "outputs";

      yield* state.deleteStack({ stack: STACK, stage });

      yield* Effect.gen(function* () {
        expect(yield* state.getOutput({ stack: STACK, stage })).toBeUndefined();

        yield* state.setOutput({
          stack: STACK,
          stage,
          value: { url: "https://example.com" },
        });
        expect(yield* state.getOutput({ stack: STACK, stage })).toEqual({
          url: "https://example.com",
        });

        // the output bookkeeping object must not leak into list()
        expect(yield* state.list({ stack: STACK, stage })).toEqual([]);

        yield* state.deleteStack({ stack: STACK, stage });
        expect(yield* state.getOutput({ stack: STACK, stage })).toBeUndefined();
      }).pipe(Effect.ensuring(cleanStage(state, stage)));
      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

test.provider(
  "getReplacedResources returns only replaced resources",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(AWS.S3.Bucket("StateBucket", { forceDestroy: true }));
      const state = yield* makeS3State({
        bucketName: bucket.bucketName,
        prefix: "test-state",
      });
      const stage = "replaced";

      yield* state.deleteStack({ stack: STACK, stage });

      yield* Effect.gen(function* () {
        const created = resource("Created", { value: "created" });
        const replaced = {
          ...resource("Replaced", { value: "replaced" }),
          status: "replaced",
        } as ResourceState;

        yield* state.set({
          stack: STACK,
          stage,
          fqn: created.fqn,
          value: created,
        });
        yield* state.set({
          stack: STACK,
          stage,
          fqn: replaced.fqn,
          value: replaced,
        });

        const result = yield* state.getReplacedResources({
          stack: STACK,
          stage,
        });
        expect(result.map((r) => r.fqn)).toEqual(["Replaced"]);
      }).pipe(Effect.ensuring(cleanStage(state, stage)));
      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

test.provider(
  "a second S3 state store cannot write a stage another deploy holds",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(AWS.S3.Bucket("StateBucket", { forceDestroy: true }));
      const options = { bucketName: bucket.bucketName, prefix: "lease" };
      const stage = "race";
      const created = resource("Vpc", { vpcId: "vpc-aaaa" });
      const request = { stack: STACK, stage, fqn: created.fqn, value: created };

      yield* Effect.scoped(
        Effect.gen(function* () {
          const held = yield* makeS3State(options);
          yield* held.set(request);
          const other = yield* makeS3State(options);
          const error = yield* other
            .set({ ...request, value: { ...created, attr: { vpcId: "vpc-bbbb" } } })
            .pipe(Effect.flip);
          expect(error).toBeInstanceOf(StateStoreError);
          expect(error.message).toContain("another deploy holds the S3 state lock");
          expect(yield* held.get({ stack: STACK, stage, fqn: created.fqn })).toMatchObject({
            attr: { vpcId: "vpc-aaaa" },
          });
          // The lease object is not a resource row.
          expect(yield* held.list({ stack: STACK, stage })).toEqual([created.fqn]);
        }),
      );

      // The first store's scope has closed, so the lease is expired and the
      // next deploy of this stage can write.
      yield* Effect.scoped(
        Effect.gen(function* () {
          const next = yield* makeS3State(options);
          yield* next.set({ ...request, value: { ...created, attr: { vpcId: "vpc-cccc" } } });
          expect(yield* next.get({ stack: STACK, stage, fqn: created.fqn })).toMatchObject({
            attr: { vpcId: "vpc-cccc" },
          });
          yield* next.deleteStack({ stack: STACK, stage });
        }),
      );

      // Closing the store deletes its lease, so a destroyed stage is no
      // longer listed.
      const after = yield* makeS3State(options);
      expect(yield* after.listStages(STACK)).not.toContain(stage);
      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

test.provider(
  "an expired S3 state lease can be taken and a live one cannot",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(AWS.S3.Bucket("StateBucket", { forceDestroy: true }));
      const prefix = "lease-expiry";
      const state = yield* makeS3State({ bucketName: bucket.bucketName, prefix });
      const expiredStage = "expired";
      const heldStage = "held";
      const leaseKey = (stage: string) => `${prefix}/${STACK}/${stage}/__lease__.json`;
      const putLease = (stage: string, expiresAt: number) =>
        s3.putObject({
          Bucket: bucket.bucketName,
          Key: leaseKey(stage),
          Body: JSON.stringify({ token: "foreign", expiresAt }),
          ContentType: "application/json",
        });

      yield* putLease(expiredStage, 0);
      const created = resource("Group", { groupId: "sg-aaaa" });
      yield* state.set({
        stack: STACK,
        stage: expiredStage,
        fqn: created.fqn,
        value: created,
      });
      expect(yield* state.list({ stack: STACK, stage: expiredStage })).toEqual([created.fqn]);

      yield* putLease(heldStage, Date.now() + 60_000);
      const blocked = yield* state
        .set({
          stack: STACK,
          stage: heldStage,
          fqn: created.fqn,
          value: created,
        })
        .pipe(Effect.flip);
      expect(blocked).toBeInstanceOf(StateStoreError);
      expect(blocked.message).toContain("another deploy holds the S3 state lock");

      yield* s3.deleteObject({ Bucket: bucket.bucketName, Key: leaseKey(heldStage) });
      yield* state.deleteStack({ stack: STACK, stage: expiredStage });
      yield* state.deleteStack({ stack: STACK, stage: heldStage });
      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

// A large deploy can queue its lease refresh behind thousands of state
// writes until the lease lapses. Nobody else took it, so the deploy keeps
// going; a deploy that did take it wins.
test.provider(
  "a lease that lapsed while still held is taken back unless another deploy took it",
  (stack) =>
    Effect.gen(function* () {
      yield* stack.destroy();
      const bucket = yield* stack.deploy(AWS.S3.Bucket("StateBucket", { forceDestroy: true }));
      const options = { bucketName: bucket.bucketName, prefix: "lease-lapse" };
      const stage = "lapse";
      const key = `lease-lapse/${STACK}/${stage}/__lease__.json`;
      const created = resource("Queue", { queueUrl: "https://example.com/a" });
      const request = { stack: STACK, stage, fqn: created.fqn, value: created };
      const lapse = Effect.gen(function* () {
        const object = yield* s3.getObject({ Bucket: bucket.bucketName, Key: key });
        const lease = JSON.parse(yield* Stream.mkString(Stream.decodeText(object.Body!)));
        yield* s3.putObject({
          Bucket: bucket.bucketName,
          Key: key,
          Body: JSON.stringify({ ...lease, expiresAt: 0 }),
          ContentType: "application/json",
        });
      });
      // Past the store's 5s trust in its last lease check.
      const pastCheck = Effect.sleep("6 seconds");

      yield* Effect.scoped(
        Effect.gen(function* () {
          const held = yield* makeS3State(options);
          yield* held.set(request);
          yield* lapse;
          yield* pastCheck;
          yield* held.set({ ...request, value: { ...created, attr: { queueUrl: "b" } } });
          expect(yield* held.get({ stack: STACK, stage, fqn: created.fqn })).toMatchObject({
            attr: { queueUrl: "b" },
          });

          // Lapsed again, and this time another deploy takes it.
          yield* lapse;
          yield* Effect.scoped(
            Effect.gen(function* () {
              const other = yield* makeS3State(options);
              yield* other.list({ stack: STACK, stage });
              yield* pastCheck;
              const lost = yield* held.set(request).pipe(Effect.flip);
              expect(lost.message).toContain("was lost mid-run");
              yield* other.deleteStack({ stack: STACK, stage });
            }),
          );
        }),
      );
      yield* stack.destroy();
    }),
  {
    tags: ["provider:aws", "provider:aws:s3", "provider:aws:statestore", "live"],
    timeout: 120_000,
  },
);

// Each deploy/destroy of a Stack declaring the S3 state store releases its
// lease when it finishes, so a file can redeploy and destroy that Stack.
const s3Stack = { providers: AWS.providers(), state: AWS.state({ backend: "s3" }) };
const s3Harness = Test.make(s3Stack);

const S3StateStack = Alchemy.Stack(
  "S3StateHarness",
  s3Stack,
  Effect.gen(function* () {
    const queue = yield* AWS.SQS.Queue("HarnessQueue");
    return { queueUrl: queue.queueUrl };
  }),
);

describe(
  "a Stack with the S3 state store",
  { tags: ["provider:aws", "provider:aws:sqs", "provider:aws:statestore", "live"] },
  () => {
    const deployed = s3Harness.beforeAll(s3Harness.deploy(S3StateStack), { timeout: 120_000 });

    s3Harness.afterAll.skipIf(!!process.env.NO_DESTROY)(
      Effect.gen(function* () {
        const { queueUrl } = yield* deployed;
        yield* s3Harness.destroy(S3StateStack);
        const gone = yield* Core.withProviders(
          sqs.getQueueAttributes({ QueueUrl: queueUrl, AttributeNames: ["QueueArn"] }).pipe(
            Effect.as(false),
            Effect.catchTag("QueueDoesNotExist", () => Effect.succeed(true)),
          ),
          s3Stack,
          "S3StateHarness",
        );
        expect(gone).toBe(true);
      }),
      { timeout: 120_000 },
    );

    s3Harness.test(
      "redeploys after the file's first deploy released its lease",
      Effect.gen(function* () {
        const { queueUrl } = yield* deployed;
        const redeployed = yield* s3Harness.deploy(S3StateStack);
        expect(redeployed.queueUrl).toBe(queueUrl);
      }),
      { timeout: 120_000 },
    );
  },
);