tests, scripts: cleanup tests and dev environment things

.github/workflows/test.yml

@@ -10,38 +10,12 @@ on:
     branches:
       - main
 
-jobs:
-  unit-tests:
-    name: Unit Tests
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    if: ${{ github.event.action == 'opened' || github.event.action == 'ready_for_review' || github.event.action == 'synchronize' || (github.event.action == 'labeled' && github.event.label.name == 'force ci') }}
-
-    steps:
-      - uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
-        with:
-          egress-policy: block
-          allowed-endpoints: >
-            api.github.com:443
-            github.com:443
-            registry.npmjs.org:443
+permissions:
+  contents: read
 
-      - name: Git Checkout
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
-
-      - name: Setup Node
-        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
-        with:
-          node-version: lts/*
-          cache: 'npm'
-
-      - name: Install dependencies
-        run: npm install
-
-      - name: Run Tests
-        run: node --run test:unit
-  e2e-test:
-    name: E2E Tests
+jobs:
+  tests:
+    name: Tests
     runs-on: ubuntu-latest
     timeout-minutes: 5
     if: ${{ github.event.action == 'opened' || github.event.action == 'ready_for_review' || github.event.action == 'synchronize' || (github.event.action == 'labeled' && github.event.label.name == 'force ci') }}
@@ -69,4 +43,4 @@ jobs:
         run: npm install
 
       - name: Run Tests
-        run: node --run test:e2e
+        run: node --run test

COLLABORATOR_GUIDE.md

@@ -62,32 +62,36 @@ The Worker also uses several other Open Source libraries (not limited to) listed
 
 ### Structure of this Repository
 
-- `src/` - Worker
-- `scripts/` - Multi-purpose scripts
-- `tests/` - Tests for the worker
+- `dev-bucket/` - A recreation of the contents in the R2 bucket that the worker reads from
+- `docs/` - Documentation on things relating to the worker
+- `e2e-tests/` - End-to-End tests for the worker
+- `scripts/` - Miscellaneous scripts
+- `src/` - Code for the worker
 
 ## Testing
 
-Each new feature or bug fix should be accompanied by a unit or E2E test (when valuable). We use Node's test runner and Miniflare for our E2E tests.
+Each new feature or bug fix should be accompanied by a unit and/or E2E test (when valuable).
+We use [Vitest](https://vitest.dev/guide/) for both.
 
 ### Unit Testing
 
-Unit Tests are fundamental to ensure that code changes do not disrupt the functionalities of the Worker:
+Unit tests are important to ensure that individual parts of the Worker are acting as expected.
 
-- Unit Tests should be written as `.test.ts` files in the [tests/unit/](./tests/unit/) directory.
+- They should be written in the same directory as the file that is being tested.
+- They should be named `<name of tested file>.test.ts`.
 - They should cover utilities as well as any component that can be broken down and individually tested.
-- We utilize Node's [Test Runner](https://nodejs.org/api/test.html) and [Assert](https://nodejs.org/api/assert.html) APIs.
-- External services used in the worker should be mocked with Undici. If the service cannot be mocked with Undici, it should be mocked with a HTTP server via Node's `createServer` API.
+- They should make use of [Vitest's APIs](https://vitest.dev/guide/).
+- External services used in the worker should be mocked with Undici.
 
 ### End-to-End Testing
 
-E2E Tests are fundamental to ensure that requests made to the worker behave as expected:
+E2E tests are important to ensure that requests made to the worker as a whole behave as expected.
 
-- E2E Tests should be written as `.test.ts` files in the [tests/e2e/](./tests/e2e/) directory.
-- They should cover the various contexts of a request that could be sent to the worker from an external client.
-- We utilize Node's [Test Runner](https://nodejs.org/api/test.html) and [Assert](https://nodejs.org/api/assert.html) APIs.
-- A local version of the Worker is ran with Miniflare.
-- Like Unit Tests, any external services should be mocked for these tests as well.
+- They should be written as `.test.ts` files in the [e2e-tests/](./e2e-tests/) directory.
+- They should cover the various contexts in which a request could be sent to the worker from an external client.
+- We utilitize [Cloudflare's Vitest Integration](https://developers.cloudflare.com/workers/testing/vitest-integration/) to run a local version of the worker for these.
+- The contents of the [dev-bucket/](./dev-bucket/) folder are read and put into an in-memory R2 bucket mock for these tests. Anything in that directory is available in a test.
+- External services used in the worker should be mocked with Undici.
 
 ## Remarks on Structure and Background
 

CONTRIBUTING.md

@@ -83,12 +83,11 @@ The steps below will give you a general idea of how to prepare your local enviro
     git merge upstream/main
     ```
 
-9.  Run the following to confirm that linting and formatting are passing.
+9.  Run the following to confirm that linting, formatting, and tests are passing.
 
     ```bash
     node --run format
-    node --run test:unit
-    node --run test:e2e
+    node --run test
     ```
 
 10. To run the worker locally, see [Dev Setup](./docs/dev-setup.md).
@@ -115,8 +114,8 @@ This repository contains a few scripts and commands for performing numerous task
 
 - `node --run format` Formats the code to the repository's standards.
 - `node --run lint` Lints the code to the repository's standards.
-- `node --run test:unit` Runs the [Unit Tests](./COLLABORATOR_GUIDE.md#unit-tests) to ensure individual components are working as expected.
-- `node --run test:e2e` Runs the [E2E Tests](./COLLABORATOR_GUIDE.md#e2e-tests) to ensure requests act as expected.
+- `node --run test` Run all tests (denoted by the `.test.ts` file extension) once
+- `node --run test:watch` Run all tests (denoted by the `.test.ts` file extension) once, then again on any file changes.
 - `node --run build:mustache` Compiles the Mustache templates. **Required for any changes to the templates to take affect**.
 
 </details>

dev-bucket/README.md

@@ -0,0 +1,6 @@
+# dev-bucket
+
+This is a very slimmed down recreation of the `dist-prod` bucket meant for testing the Release Worker.
+The files here have little to no content, they're here just to build the structure of the bucket itself so that the Release Worker can be used in development easier.
+
+For more information on the structure of the bucket, read [R2](../docs/r2.md) and [Release Process](../docs/release-process.md).

dev-bucket/metrics/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>metrics/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/chakracore-nightly/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/chakracore-nightly/v10.13.0-nightly2018112084bd6f3c82/docs/api/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/chakracore-nightly/v10.13.0-nightly2018112084bd6f3c82/docs/api/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/chakracore-rc/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/chakracore-release/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/chakracore-release/v10.0.0/docs/api/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/chakracore-release/v10.0.0/docs/api/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/docs/v0.0.1/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/docs/v0.0.1/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/nightly/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/nightly/v24.0.0-nightly20241219756077867b/docs/api/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/nightly/v24.0.0-nightly20241219756077867b/docs/api/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/nightly/v24.0.0-nightly20241219756077867b/docs/apilinks.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/rc/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/rc/v23.0.0-rc.3/docs/api/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/rc/v23.0.0-rc.3/docs/api/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/rc/v23.0.0-rc.3/docs/apilinks.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/release/index.json

@@ -0,0 +1 @@
+{ "hello": true }

dev-bucket/nodejs/release/v20.0.0/docs/api/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/release/v20.0.0/docs/api/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/release/v20.0.0/docs/apilinks.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/test/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/v8-canary/index.json

@@ -0,0 +1 @@
+{}

dev-bucket/nodejs/v8-canary/v24.0.0-v8-canary202412221f947c1730/docs/api/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Document</title>
+</head>
+<body>
+    <p>nodejs/v8-canary/v24.0.0-v8-canary202412221f947c1730/docs/api/index.html</p>
+</body>
+</html>

dev-bucket/nodejs/v8-canary/v24.0.0-v8-canary202412221f947c1730/docs/apilinks.json

@@ -0,0 +1 @@
+{}

e2e-tests/README.md

@@ -0,0 +1,10 @@
+# e2e-tests
+
+End-to-End tests for the Release Worker.
+
+## Table of Contents
+
+- [file.test.ts](./file.test.ts) - Tests for requests specific to files.
+- [directory.test.ts](./directory.test.ts) - Tests for requests specific to directories.
+- [fallback.test.ts](./fallback.test.ts) - Tests for falling back to the origin server if needed.
+- [misc.test.ts](./misc.test.ts) - Miscellaneous tests not pertaining specifically to files or directories.

e2e-tests/directory.test.ts

@@ -0,0 +1,137 @@
+import { env, fetchMock, createExecutionContext } from 'cloudflare:test';
+import { test, beforeAll, afterEach, expect } from 'vitest';
+import { populateR2WithDevBucket } from './util';
+import worker from '../src/worker';
+import type { Env } from '../src/env';
+import { CACHE_HEADERS } from '../src/constants/cache';
+
+const mockedEnv: Env = {
+  ...env,
+  ENVIRONMENT: 'e2e-tests',
+  CACHING: false,
+  LOG_ERRORS: true,
+  S3_ENDPOINT: 'https://s3.mock',
+  S3_ACCESS_KEY_ID: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+  S3_ACCESS_KEY_SECRET:
+    'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
+};
+
+const s3Url = new URL(mockedEnv.S3_ENDPOINT);
+s3Url.host = `${mockedEnv.BUCKET_NAME}.${s3Url.host}`;
+
+const S3_DIRECTORY_RESULT = `
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+  <Name>dist-prod</Name>
+  <Prefix />
+  <Marker />
+  <MaxKeys>1000</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+  <CommonPrefixes>
+    <Prefix>nodejs/release/v1.0.0/latest/</Prefix>
+  </CommonPrefixes>
+  <Contents>
+    <ETag>"asd123"</ETag>
+    <Key>nodejs/release/v1.0.0/index.json</Key>
+    <LastModified>2023-09-12T05:43:00.000Z</LastModified>
+    <Size>18</Size>
+  </Contents>
+</ListBucketResult>`;
+
+const S3_EMPTY_DIRECTORY_RESULT = `
+<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
+  <Name>dist-prod</Name>
+  <Prefix />
+  <Marker />
+  <MaxKeys>1000</MaxKeys>
+  <IsTruncated>false</IsTruncated>
+</ListBucketResult>`;
+
+beforeAll(async () => {
+  fetchMock.activate();
+  fetchMock.disableNetConnect();
+
+  await populateR2WithDevBucket();
+});
+
+afterEach(() => {
+  fetchMock.assertNoPendingInterceptors();
+});
+
+// These paths are cached and don't send requests to S3
+for (const path of ['/dist/', '/docs/']) {
+  test(`GET \`${path}\` returns 200`, async () => {
+    const ctx = createExecutionContext();
+
+    const res = await worker.fetch(
+      new Request(`https://localhost${path}`),
+      mockedEnv,
+      ctx
+    );
+
+    // Consume body promise
+    await res.text();
+
+    expect(res.status).toBe(200);
+  });
+}
+
+for (const path of ['/api/', '/download/', '/metrics/']) {
+  test(`GET \`${path}\` returns 200`, async () => {
+    fetchMock
+      .get(s3Url.origin)
+      .intercept({
+        path: /.*/,
+      })
+      .reply(200, S3_DIRECTORY_RESULT);
+
+    const ctx = createExecutionContext();
+
+    const res = await worker.fetch(
+      new Request(`https://localhost${path}`),
+      mockedEnv,
+      ctx
+    );
+
+    // Consume body promise
+    await res.text();
+
+    expect(res.status).toBe(200);
+  });
+}
+
+test('GET `/dist/unknown-directory/` returns 404', async () => {
+  fetchMock
+    .get(s3Url.origin)
+    .intercept({
+      path: /.*/,
+      query: {
+        prefix: 'nodejs/release/unknown-directory/',
+      },
+    })
+    .reply(200, S3_EMPTY_DIRECTORY_RESULT);
+
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request('https://localhost/dist/unknown-directory/'),
+    mockedEnv,
+    ctx
+  );
+
+  expect(res.status).toBe(404);
+  expect(res.headers.get('cache-control')).toStrictEqual(CACHE_HEADERS.failure);
+  expect(await res.text()).toStrictEqual('Directory not found');
+});
+
+test('GET `/dist` redirects to `/dist/`', async () => {
+  const ctx = createExecutionContext();
+
+  const res = await worker.fetch(
+    new Request('https://localhost/dist'),
+    mockedEnv,
+    ctx
+  );
+
+  expect(res.status).toBe(301);
+  expect(res.headers.get('location')).toStrictEqual('https://localhost/dist/');
+});