bug: Single Bucket Bulk Op Bugs & Tests

.github/workflows/deploy-main.yaml

@@ -42,13 +42,18 @@ jobs:
         id: deploy-worker
         uses: cloudflare/wrangler-action@v3
         with:
+          quiet: true
           preCommands: node prepare-deploy.js
           command: deploy -e stage -c wrangler-versioned.toml
           apiToken: ${{ secrets.CLOUDFLARE_AUTH }}
+          accountId: ${{secrets.CLOUDFLARE_ACCOUNT}}
       - name: Post-Deployment Integration Test
-        run: WORKER_URL="${{ steps.deploy-worker.outputs.deployment-url }}" npm run test:it
+        run: WORKER_URL="${{ steps.deploy-worker.outputs.deployment-url }}" npm run test:e2e
+        env:
+          WORKER_URL: ${{ secrets.STAGE_WORKER_URL }}
       - name: Semantic Release
         run: npm run semantic-release
         env:
           CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_AUTH }}
+          CLOUDFLARE_ACCOUNT_ID: ${{secrets.CLOUDFLARE_ACCOUNT}}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/install-lint-test.yaml

@@ -43,11 +43,15 @@ jobs:
         id: deploy-worker
         uses: cloudflare/wrangler-action@v3
         with:
+          quiet: true
           preCommands: node prepare-deploy.js
           command: deploy -e stage -c wrangler-versioned.toml
           apiToken: ${{ secrets.CLOUDFLARE_AUTH }}
+          accountId: ${{secrets.CLOUDFLARE_ACCOUNT}}
       - name: Post-Deployment Integration Test
-        run: WORKER_URL="${{ steps.deploy-worker.outputs.deployment-url }}" npm run test:it
+        run: WORKER_URL="${{ steps.deploy-worker.outputs.deployment-url }}" npm run test:e2e
+        env:
+          WORKER_URL: ${{ secrets.STAGE_WORKER_URL }}
       - name: Semantic Release (Dry Run)
         run: npm run semantic-release-dry
         env:

src/handlers/get.js

@@ -26,6 +26,7 @@ function getRobots() {
 
 export default async function getHandler({ env, daCtx }) {
   const { path } = daCtx;
+  if (path === '/') return get404();
 
   if (path.startsWith('/favicon.ico')) return get404();
   if (path.startsWith('/robots.txt')) return getRobots();

src/storage/object/copy.js

@@ -21,11 +21,10 @@ import { putObjectWithVersion } from '../version/put.js';
 import { listCommand } from '../utils/list.js';
 import { hasPermission } from '../../utils/auth.js';
 
-const MAX_KEYS = 900;
+const MAX_KEYS = 35;
 
 export const copyFile = async (config, env, daCtx, sourceKey, details, isRename) => {
   const Key = `${sourceKey.replace(details.source, details.destination)}`;
-
   if (!hasPermission(daCtx, sourceKey, 'read') || !hasPermission(daCtx, Key, 'write')) {
     return {
       $metadata: {
@@ -69,19 +68,24 @@ export const copyFile = async (config, env, daCtx, sourceKey, details, isRename)
         tags: ['METADATA', 'IF-NONE-MATCH'],
       },
     );
+
     const resp = await client.send(new CopyObjectCommand(input));
     return resp;
   } catch (e) {
     if (e.$metadata.httpStatusCode === 412) {
       // Not the happy path - something is at the destination already.
       if (!isRename) {
         // This is a copy so just put the source into the target to keep the history.
-
-        const original = await getObject(env, { org: daCtx.org, key: sourceKey });
+        const original = await getObject(
+          env,
+          { bucket: daCtx.bucket, org: daCtx.org, key: sourceKey },
+        );
+        const body = await original.body.transformToString();
         return /* await */ putObjectWithVersion(env, daCtx, {
+          bucket: daCtx.bucket,
           org: daCtx.org,
           key: Key,
-          body: original.body,
+          body,
           contentLength: original.contentLength,
           type: original.contentType,
         });
@@ -101,14 +105,13 @@ export const copyFile = async (config, env, daCtx, sourceKey, details, isRename)
   } finally {
     if (Key.endsWith('.html')) {
       // Reset the collab cached state for the copied object
-      await invalidateCollab('syncAdmin', `${daCtx.origin}/source/${daCtx.org}/${Key}`, env);
+      await invalidateCollab('syncadmin', `${daCtx.origin}/source/${daCtx.org}/${Key}`, env);
     }
   }
 };
 
 export default async function copyObject(env, daCtx, details, isRename) {
   if (details.source === details.destination) return { body: '', status: 409 };
-
   const config = getS3Config(env);
   const client = new S3Client(config);
 
@@ -125,7 +128,7 @@ export default async function copyObject(env, daCtx, details, isRename) {
       let resp = await listCommand(daCtx, details, client);
       sourceKeys = resp.sourceKeys;
       if (resp.continuationToken) {
-        continuationToken = `copy-${details.source}-${details.destination}-${crypto.randomUUID()}`;
+        continuationToken = `copy-${daCtx.org}-${details.source}-${details.destination}-${crypto.randomUUID()}`;
         while (resp.continuationToken) {
           resp = await listCommand(daCtx, { continuationToken: resp.continuationToken }, client);
           remainingKeys.push(...resp.sourceKeys);

src/storage/object/list.js

@@ -37,7 +37,6 @@ export default async function listObjects(env, daCtx, maxKeys) {
   const command = new ListObjectsV2Command(input);
   try {
     const resp = await client.send(command);
-    // console.log(resp);
     const body = formatList(resp);
     return {
       body: JSON.stringify(body),

src/storage/object/move.js

@@ -48,7 +48,7 @@ export default async function moveObject(env, daCtx, details) {
       const resp = await client.send(command);
 
       const { Contents = [], NextContinuationToken } = resp;
-      sourceKeys.push(...Contents.map(({ Key }) => Key));
+      sourceKeys.push(...Contents.map(({ Key }) => Key.replace(`${daCtx.org}/`, '')));
 
       const movedLoad = sourceKeys
         .filter((key) => hasPermission(daCtx, key, 'write'))

src/storage/object/put.js

@@ -37,21 +37,6 @@ function buildInput({
   };
 }
 
-function createBucketIfMissing(client) {
-  client.middlewareStack.add(
-    (next) => async (args) => {
-      // eslint-disable-next-line no-param-reassign
-      args.request.headers['cf-create-bucket-if-missing'] = 'true';
-      return next(args);
-    },
-    {
-      step: 'build',
-      name: 'createIfMissingMiddleware',
-      tags: ['METADATA', 'CREATE-BUCKET-IF-MISSING'],
-    },
-  );
-}
-
 /**
  * Check to see if the org is in the existing list of orgs
  *
@@ -77,9 +62,6 @@ export default async function putObject(env, daCtx, obj) {
   // Only allow creating a new bucket for orgs and repos
   if (key.split('/').length <= 1) {
     await checkOrgIndex(env, org);
-
-    // R2 ONLY FEATURE
-    createBucketIfMissing(client);
   }
 
   const inputs = [];

src/storage/utils/config.js

@@ -17,5 +17,6 @@ export default function getS3Config(env) {
       accessKeyId: env.S3_ACCESS_KEY_ID,
       secretAccessKey: env.S3_SECRET_ACCESS_KEY,
     },
+    forcePathStyle: true,
   };
 }

src/storage/utils/list.js

@@ -78,7 +78,7 @@ function buildInput(bucket, org, key) {
   return {
     Bucket: bucket,
     Prefix: `${org}/${key}/`,
-    MaxKeys: 300,
+    MaxKeys: 35,
   };
 }
 
@@ -106,7 +106,7 @@ export async function listCommand(daCtx, details, s3client) {
   const resp = await s3client.send(command);
 
   const { Contents = [], NextContinuationToken } = resp;
-  sourceKeys.push(...Contents.map(({ Key }) => Key));
+  sourceKeys.push(...Contents.map(({ Key }) => Key.replace(`${daCtx.org}/`, '')));
 
   return { sourceKeys, continuationToken: NextContinuationToken };
 }

src/storage/utils/version.js

@@ -13,25 +13,8 @@ import {
   S3Client,
 } from '@aws-sdk/client-s3';
 
-export function createBucketIfMissing(client) {
-  client.middlewareStack.add(
-    (next) => async (args) => {
-      // eslint-disable-next-line no-param-reassign
-      args.request.headers['cf-create-bucket-if-missing'] = 'true';
-      return next(args);
-    },
-    {
-      step: 'build',
-      name: 'createIfMissingMiddleware',
-      tags: ['METADATA', 'CREATE-BUCKET-IF-MISSING'],
-    },
-  );
-  return client;
-}
-
 export function ifNoneMatch(config) {
   const client = new S3Client(config);
-  createBucketIfMissing(client);
   client.middlewareStack.add(
     (next) => async (args) => {
       // eslint-disable-next-line no-param-reassign
@@ -49,7 +32,6 @@ export function ifNoneMatch(config) {
 
 export function ifMatch(config, match) {
   const client = new S3Client(config);
-  createBucketIfMissing(client);
   client.middlewareStack.add(
     (next) => async (args) => {
       // eslint-disable-next-line no-param-reassign

src/storage/version/put.js

@@ -16,7 +16,7 @@ import {
 
 import getS3Config from '../utils/config.js';
 import {
-  createBucketIfMissing, ifMatch, ifNoneMatch,
+  ifMatch, ifNoneMatch,
 } from '../utils/version.js';
 import getObject from '../object/get.js';
 
@@ -39,7 +39,7 @@ export async function putVersion(config, {
 }, noneMatch = true) {
   const length = ContentLength ?? getContentLength(Body);
 
-  const client = noneMatch ? ifNoneMatch(config) : createBucketIfMissing(new S3Client(config));
+  const client = noneMatch ? ifNoneMatch(config) : new S3Client(config);
   const input = {
     Bucket, Key: `${Org}/.da-versions/${ID}/${Version}.${Ext}`, Body, Metadata, ContentLength: length,
   };
@@ -118,7 +118,6 @@ export async function putObjectWithVersion(env, daCtx, update, body) {
       Label,
     },
   });
-
   if (versionResp.status !== 200 && versionResp.status !== 412) {
     return versionResp.status;
   }
@@ -132,7 +131,6 @@ export async function putObjectWithVersion(env, daCtx, update, body) {
   });
   try {
     const resp = await client.send(command);
-
     return resp.$metadata.httpStatusCode;
   } catch (e) {
     if (e.$metadata.httpStatusCode === 412) {

test/e2e/copy.spec.js

@@ -0,0 +1,83 @@
+
+
+import assert from 'node:assert';
+
+const workerUrl = process.env.WORKER_URL;
+const ORG = 'da-e2e-test';
+
+describe('/copy operation', function() {
+  this.timeout(0);
+  it('copies a file', async () => {
+
+    const blob = new Blob(['Hello World!'], { type: "text/html" });
+    let body = new FormData();
+    body.append('data', blob);
+    let opts = {
+      body,
+      method: 'POST'
+    };
+    let req = new Request(`${workerUrl}/source/${ORG}/copy-spec/test-file.html`, opts);
+    let resp = await fetch(req);
+    assert(resp.status === 200 || resp.status === 201);
+
+    body = new FormData();
+    body.append('destination', `${ORG}/copy-spec/test-file-copy.html`)
+    opts = {
+      body,
+      method: 'POST'
+    }
+    req = new Request(`${workerUrl}/copy/${ORG}/copy-spec/test-file.html`, opts);
+    resp = await fetch(req);
+    assert.strictEqual(resp.status, 204);
+
+    resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-file.html`);
+    assert.strictEqual(resp.status, 200);
+
+    resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-file-copy.html`);
+    assert.strictEqual(resp.status, 200);
+    const content = await resp.text();
+    assert.strictEqual(content, 'Hello World!');
+
+    resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-file.html`, { method: 'DELETE' });
+    assert.strictEqual(resp.status, 204);
+
+    resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-file-copy.html`, { method: 'DELETE' });
+    assert.strictEqual(resp.status, 204);
+  });
+
+  it('copies a folder', async () => {
+    const limit = 5;
+    for (let i = 0; i < limit; i++) {
+      const blob = new Blob(['Hello World!'], { type: "text/html" });
+      let body = new FormData();
+      body.append('data', blob);
+      let opts = {
+        body,
+        method: 'POST'
+      };
+      let req = new Request(`${workerUrl}/source/${ORG}/copy-spec/test-folder/index${i}.html`, opts);
+      let resp = await fetch(req);
+      assert(resp.status === 200 || resp.status === 201);
+    }
+    const body = new FormData();
+    body.append('destination', `/${ORG}/copy-spec/test-folder-copy`);
+    const opts = {
+      body,
+      method: 'POST',
+    };
+    const req = new Request(`${workerUrl}/copy/${ORG}/copy-spec/test-folder`, opts);
+    const resp = await fetch(req);
+    assert.strictEqual(resp.status, 204);
+    for (let i = 0; i < limit; i++) {
+      let resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-folder/index${i}.html`);
+      assert.strictEqual(resp.status, 200);
+      resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-folder-copy/index${i}.html`);
+      assert.strictEqual(resp.status, 200);
+
+      resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-folder/index${i}.html`, { method: 'DELETE' });
+      assert.strictEqual(resp.status, 204);
+      resp = await fetch(`${workerUrl}/source/${ORG}/copy-spec/test-folder-copy/index${i}.html`, { method: 'DELETE' });
+      assert.strictEqual(resp.status, 204);
+    }
+  });
+});

test/e2e/list.spec.js

@@ -0,0 +1,37 @@
+import assert from 'node:assert';
+
+const workerUrl = process.env.WORKER_URL;
+
+const ORG = 'da-e2e-test'
+
+describe('/list requests', function () {
+  this.timeout(0);
+  it('returns orgs', async () => {
+    const req = new Request(`${workerUrl}/list`);
+    const resp = await fetch(req);
+    assert.strictEqual(resp.status, 200);
+    const body = await resp.json();
+    assert(body.length > 0);
+  });
+
+  it('returns content', async () => {
+    const blob = new Blob(['Hello World!'], { type: "text/html" });
+    const body = new FormData();
+    body.append('data', blob);
+    const opts =  {
+      body,
+      method: 'POST'
+    };
+    let req = new Request(`${workerUrl}/source/${ORG}/list-spec/test-file.html`, opts);
+    let resp = await fetch(req);
+    assert(resp.status === 200 || resp.status === 201);
+    req = new Request(`${workerUrl}/list/${ORG}/list-spec`);
+    resp = await fetch(req);
+    assert.strictEqual(resp.status, 200);
+    const json = await resp.json();
+    assert(json.some(org => org.path === `/${ORG}/list-spec/test-file.html`));
+
+    resp = await fetch(`${workerUrl}/source/${ORG}/list-spec/test-file.html`, { method: 'DELETE' });
+    assert.strictEqual(resp.status, 204);
+  });
+});