sailfork / t3code-github / .repos/ alchemy-effect/ packages/ integration-tests/ test/ GCP/ StorageTransfer/ TransferJob.test.ts

import * as GCP from "@alchemy.run/gcp";
import { GcpEnvironment } from "@alchemy.run/gcp/Environment";
import { expect } from "@alchemy.run/test";
import * as Test from "@alchemy.run/test/Alchemy";
import * as storage from "@distilled.cloud/gcp/storage_v1";
import * as storagetransfer from "@distilled.cloud/gcp/storagetransfer_v1";
import * as Effect from "effect/Effect";
import { MinimumLogLevel } from "effect/References";
import * as Schedule from "effect/Schedule";

const { test } = Test.make({ providers: GCP.providers() });

const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");

const waitUntilGone = (jobName: string) =>
  Effect.gen(function* () {
    const { project } = yield* GcpEnvironment.current;

    return yield* storagetransfer.getTransferJobs({ jobName, projectId: project }).pipe(
      Effect.map((job) => (job.status === "DELETED" ? ("gone" as const) : ("found" as const))),
      Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
      Effect.repeat({
        schedule: Schedule.spaced("1 second"),
        until: (status) => status === "gone",
        times: 10,
      }),
    );
  });

const grantBucketRole = (bucket: string, member: string, role: string) =>
  Effect.gen(function* () {
    const policy = yield* storage.getIamPolicyBuckets({
      bucket,
      optionsRequestedPolicyVersion: 3,
    });
    const bindings = [...(policy.bindings ?? [])];
    const existing = bindings.find((binding) => binding.role === role);
    if (existing?.members?.includes(member)) return;
    if (existing) {
      existing.members = [...(existing.members ?? []), member];
    } else {
      bindings.push({ role, members: [member] });
    }
    yield* storage.setIamPolicyBuckets({
      bucket,
      body: {
        ...policy,
        bindings,
      },
    });
  }).pipe(
    Effect.retry({
      while: (error) => error._tag === "Conflict",
      times: 4,
      schedule: Schedule.spaced("500 millis"),
    }),
  );

const grantTransferAccess = (sourceBucket: string, sinkBucket: string, email: string) => {
  const member = `serviceAccount:${email}`;
  return Effect.all(
    [
      grantBucketRole(sourceBucket, member, "roles/storage.legacyBucketReader"),
      grantBucketRole(sourceBucket, member, "roles/storage.objectViewer"),
      grantBucketRole(sinkBucket, member, "roles/storage.legacyBucketReader"),
      grantBucketRole(sinkBucket, member, "roles/storage.objectAdmin"),
    ],
    { concurrency: 1 },
  );
};

test.provider(
  "create, update, and delete a transfer job",
  (stack) =>
    Effect.gen(function* () {
      const { project } = yield* GcpEnvironment.current;

      yield* stack.destroy();

      const buckets = yield* stack.deploy(
        Effect.gen(function* () {
          const source = yield* GCP.Storage.Bucket("Src", {
            location: "US-CENTRAL1",
            forceDestroy: true,
          });
          const sink = yield* GCP.Storage.Bucket("Dst", {
            location: "US-CENTRAL1",
            forceDestroy: true,
          });
          return {
            sourceBucket: source.bucketName.as<string>(),
            sinkBucket: sink.bucketName.as<string>(),
          };
        }),
      );

      const account = yield* storagetransfer.getGoogleServiceAccounts({
        projectId: project,
      });
      if (account.accountEmail) {
        yield* grantTransferAccess(buckets.sourceBucket, buckets.sinkBucket, account.accountEmail);
      }

      const created = yield* stack.deploy(
        Effect.gen(function* () {
          yield* GCP.Storage.Bucket("Src", {
            bucketName: buckets.sourceBucket,
            location: "US-CENTRAL1",
            forceDestroy: true,
          });
          yield* GCP.Storage.Bucket("Dst", {
            bucketName: buckets.sinkBucket,
            location: "US-CENTRAL1",
            forceDestroy: true,
          });
          return yield* GCP.StorageTransfer.TransferJob("Copy", {
            description: "copy uploads",
            status: "DISABLED",
            transferSpec: {
              gcsDataSource: { bucketName: buckets.sourceBucket },
              gcsDataSink: { bucketName: buckets.sinkBucket },
            },
          });
        }),
      );

      expect(created.jobId).toEqual(expect.any(String));
      expect(created.name).toEqual(`transferJobs/${created.jobId}`);
      expect(created.project).toEqual(project);
      expect(created.description).toEqual("copy uploads");
      expect(created.status).toEqual("DISABLED");
      expect(created.transferSpec?.gcsDataSource?.bucketName).toEqual(buckets.sourceBucket);
      expect(created.transferSpec?.gcsDataSink?.bucketName).toEqual(buckets.sinkBucket);

      const fetched = yield* storagetransfer.getTransferJobs({
        jobName: created.name,
        projectId: project,
      });
      expect(fetched.name).toEqual(created.name);
      expect(fetched.status).toEqual("DISABLED");
      expect(fetched.description).toContain("alchemy-id=");
      expect(fetched.description).toContain("copy uploads");
      expect(fetched.transferSpec?.gcsDataSource?.bucketName).toEqual(buckets.sourceBucket);

      const updated = yield* stack.deploy(
        Effect.gen(function* () {
          yield* GCP.Storage.Bucket("Src", {
            bucketName: buckets.sourceBucket,
            location: "US-CENTRAL1",
            forceDestroy: true,
          });
          yield* GCP.Storage.Bucket("Dst", {
            bucketName: buckets.sinkBucket,
            location: "US-CENTRAL1",
            forceDestroy: true,
          });
          return yield* GCP.StorageTransfer.TransferJob("Copy", {
            jobId: created.jobId,
            description: "copy inbox",
            status: "DISABLED",
            transferSpec: {
              gcsDataSource: {
                bucketName: buckets.sourceBucket,
                path: "inbox/",
              },
              gcsDataSink: { bucketName: buckets.sinkBucket },
            },
          });
        }),
      );

      expect(updated.name).toEqual(created.name);
      expect(updated.jobId).toEqual(created.jobId);
      expect(updated.description).toEqual("copy inbox");
      expect(updated.transferSpec?.gcsDataSource?.path).toEqual("inbox/");

      const fetchedUpdate = yield* storagetransfer.getTransferJobs({
        jobName: created.name,
        projectId: project,
      });
      expect(fetchedUpdate.description).toContain("copy inbox");
      expect(fetchedUpdate.transferSpec?.gcsDataSource?.path).toEqual("inbox/");
      expect(fetchedUpdate.status).toEqual("DISABLED");

      yield* stack.destroy();

      const gone = yield* waitUntilGone(created.name);
      expect(gone).toEqual("gone");
    }).pipe(logLevel),
  {
    tags: ["provider:gcp", "provider:gcp:storagetransfer", "live"],
    timeout: 120_000,
  },
);