Refactor storage to use DriveManager, enabling testing

.env.sample

@@ -18,12 +18,17 @@ ALLOW_PRIVATE_ADDRESS=false
 
 REMOTE_ACTOR_FETCH_POSTS=10
 
+# File uploads and media storage:
 DRIVE_DISK=
-ASSET_URL_BASE=
-FS_ASSET_PATH=
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-S3_REGION=
-S3_ENDPOINT_URL=
-S3_BUCKET=
-S3_FORCE_PATH_STYLE=false
+STORAGE_URL_BASE=
+
+# If DRIVE_DISK is "fs":
+# FS_STORAGE_PATH=
+
+# If DRIVE_DISK is "s3":
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# S3_REGION=
+# S3_ENDPOINT_URL=
+# S3_BUCKET=
+# S3_FORCE_PATH_STYLE=false

.env.test

@@ -1,8 +1,8 @@
 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/hollo_test
 SECRET_KEY="test_determinist_key_DO_NOT_USE_IN_PRODUCTION"
 
-# LOG_LEVEL=debug
-# LOG_QUERY=true
+LOG_LEVEL=debug
+LOG_QUERY=true
 
 # Setting ALLOW_PRIVATE_ADDRESS to true disables SSRF (Server-Side Request Forgery) protection
 # Set to true to test in local network
@@ -14,5 +14,5 @@ REMOTE_ACTOR_FETCH_POSTS=10
 
 # We actually use fake storage in tests:
 DRIVE_DISK=fs
-FS_ASSET_PATH=tmp/test_storage
-ASSET_URL_BASE="http://hollo.test/"
+FS_STORAGE_PATH=tmp/fakes
+STORAGE_URL_BASE="http://hollo.test/"

.github/workflows/test.yaml

@@ -56,8 +56,10 @@ jobs:
       # Will be replaced by list of allowed IPs once https://github.com/dahlia/fedify/issues/157
       # is implemented.
       ALLOW_PRIVATE_ADDRESS: false
+
       DRIVE_DISK: "fs"
-      FS_ASSET_PATH: ./tmp/test_storage
+      FS_STORAGE_PATH: ./tmp/fakes
+      STORAGE_URL_BASE: "http://hollo.test/"
     steps:
       - uses: actions/checkout@v4
       - uses: pnpm/action-setup@v4
@@ -69,8 +71,6 @@ jobs:
           node-version: 23
           cache: pnpm
       - run: pnpm install
-      - name: Ensure the directory for test uploads
-        run: mkdir -p tmp/test_storage
       - name: Run the database migrations
         run: pnpm run migrate
       - name: Run the tests

.gitignore

@@ -3,3 +3,4 @@ assets/
 fedify-hollo-*.tgz
 *.jsonl
 node_modules/
+tmp/

compose-fs.yaml

@@ -9,8 +9,8 @@ services:
       LOG_LEVEL: "${LOG_LEVEL}"
       BEHIND_PROXY: "${BEHIND_PROXY}"
       DRIVE_DISK: fs
-      ASSET_URL_BASE: http://localhost:3000/assets/
-      FS_ASSET_PATH: /var/lib/hollo
+      STORAGE_URL_BASE: http://localhost:3000/assets/
+      FS_STORAGE_PATH: /var/lib/hollo
     depends_on:
       - postgres
     volumes:

compose.yaml

@@ -2,24 +2,24 @@ services:
   hollo:
     image: ghcr.io/fedify-dev/hollo:canary
     ports:
-    - "3000:3000"
+      - "3000:3000"
     environment:
       DATABASE_URL: "postgres://user:password@postgres:5432/database"
       SECRET_KEY: "${SECRET_KEY}"
       LOG_LEVEL: "${LOG_LEVEL}"
       BEHIND_PROXY: "${BEHIND_PROXY}"
       DRIVE_DISK: s3
-      ASSET_URL_BASE: http://localhost:9000/hollo/
+      STORAGE_URL_BASE: http://localhost:9000/hollo/
       S3_REGION: us-east-1
       S3_BUCKET: hollo
       S3_ENDPOINT_URL: http://minio:9000
       S3_FORCE_PATH_STYLE: "true"
       AWS_ACCESS_KEY_ID: minioadmin
       AWS_SECRET_ACCESS_KEY: minioadmin
     depends_on:
-    - postgres
-    - minio
-    - create-bucket
+      - postgres
+      - minio
+      - create-bucket
     restart: unless-stopped
 
   postgres:
@@ -29,24 +29,24 @@ services:
       POSTGRES_PASSWORD: password
       POSTGRES_DB: database
     volumes:
-    - postgres_data:/var/lib/postgresql/data
+      - postgres_data:/var/lib/postgresql/data
     restart: unless-stopped
 
   minio:
     image: minio/minio:RELEASE.2024-09-13T20-26-02Z
     ports:
-    - "9000:9000"
+      - "9000:9000"
     environment:
       MINIO_ROOT_USER: minioadmin
       MINIO_ROOT_PASSWORD: minioadmin
     volumes:
-    - minio_data:/data
+      - minio_data:/data
     command: ["server", "/data", "--console-address", ":9001"]
 
   create-bucket:
     image: minio/mc:RELEASE.2024-09-16T17-43-14Z
     depends_on:
-    - minio
+      - minio
     entrypoint: |
       /bin/sh -c "
         /usr/bin/mc alias set minio http://minio:9000 minioadmin minioadmin;

package.json

@@ -8,7 +8,7 @@
     "prod": "pnpm run migrate && tsx --env-file-if-exists=.env --dns-result-order=ipv6first bin/server.ts",
     "dev": "pnpm run migrate && tsx watch --env-file-if-exists=.env --dns-result-order=ipv6first bin/server.ts",
     "test": "pnpm run migrate:test && tsx --env-file-if-exists=.env.test --test",
-    "test:ci": "drizzle-kit migrate && tsx --test --test-reporter=@reporters/github --test-reporter-destination=stdout",
+    "test:ci": "tsx --test --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=spec --test-reporter-destination=stdout",
     "check": "tsc && biome check .",
     "check:ci": "tsc && biome ci --reporter=github .",
     "migrate": "drizzle-kit migrate",

src/api/v1/accounts.ts

@@ -53,7 +53,7 @@ import {
   pinnedPosts,
   posts,
 } from "../../schema";
-import { disk, getAssetUrl } from "../../storage";
+import { drive } from "../../storage";
 import { extractCustomEmojis, formatText } from "../../text";
 import { type Uuid, isUuid } from "../../uuid";
 import { timelineQuerySchema } from "./timelines";
@@ -107,6 +107,7 @@ app.patch(
     }),
   ),
   async (c) => {
+    const disk = drive.use();
     const owner = c.get("token").accountOwner;
     if (owner == null) {
       return c.json(
@@ -133,7 +134,7 @@ app.patch(
         contentLength: content.byteLength,
         visibility: "public",
       });
-      avatarUrl = getAssetUrl(`${path}?${Date.now()}`, c.req.url);
+      avatarUrl = await disk.getUrl(path);
     }
     let coverUrl = undefined;
     if (form.header instanceof File) {
@@ -156,7 +157,7 @@ app.patch(
       } catch (error) {
         return c.json({ error: "Failed to upload header image." }, 500);
       }
-      coverUrl = getAssetUrl(`${path}?${Date.now()}`, c.req.url);
+      coverUrl = await disk.getUrl(path);
     }
     const fedCtx = federation.createContext(c.req.raw, undefined);
     const fmtOpts = {

src/api/v1/media.ts

@@ -7,12 +7,13 @@ import { serializeMedium } from "../../entities/medium";
 import { makeVideoScreenshot, uploadThumbnail } from "../../media";
 import { type Variables, scopeRequired, tokenRequired } from "../../oauth";
 import { media } from "../../schema";
-import { disk, getAssetUrl } from "../../storage";
+import { drive } from "../../storage";
 import { isUuid, uuidv7 } from "../../uuid";
 
 const app = new Hono<{ Variables: Variables }>();
 
 export async function postMedia(c: Context<{ Variables: Variables }>) {
+  const disk = drive.use();
   const owner = c.get("token").accountOwner;
   if (owner == null) {
     return c.json({ error: "This method requires an authenticated user" }, 422);
@@ -47,7 +48,7 @@ export async function postMedia(c: Context<{ Variables: Variables }>) {
   } catch (error) {
     return c.json({ error: "Failed to save media file" }, 500);
   }
-  const url = getAssetUrl(path, c.req.url);
+  const url = await disk.getUrl(path);
   const result = await db
     .insert(media)
     .values({
@@ -57,7 +58,7 @@ export async function postMedia(c: Context<{ Variables: Variables }>) {
       width: fileMetadata.width!,
       height: fileMetadata.height!,
       description,
-      ...(await uploadThumbnail(id, image, c.req.url)),
+      ...(await uploadThumbnail(id, image)),
     })
     .returning();
   if (result.length < 1) {

src/federation/post.ts

@@ -416,7 +416,7 @@ export async function persistPost(
       }
       const image = sharp(imageBytes);
       metadata = await image.metadata();
-      thumbnail = await uploadThumbnail(id, image, baseUrl);
+      thumbnail = await uploadThumbnail(id, image);
     } catch {
       metadata = {
         width: attachment.width ?? 512,

src/index.tsx

@@ -10,7 +10,7 @@ import fedi from "./federation";
 import image from "./image";
 import oauth, { oauthAuthorizationServer } from "./oauth";
 import pages from "./pages";
-import { DRIVE_DISK, assetPath } from "./storage";
+import { DRIVE_DISK, FS_STORAGE_PATH } from "./storage";
 
 const app = new Hono();
 
@@ -23,7 +23,7 @@ if (DRIVE_DISK === "fs") {
   app.use(
     "/assets/*",
     serveStatic({
-      root: relative(process.cwd(), assetPath!),
+      root: relative(process.cwd(), FS_STORAGE_PATH!),
       rewriteRequestPath: (path) => path.substring("/assets".length),
     }),
   );

src/media.ts

@@ -4,8 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import ffmpeg from "fluent-ffmpeg";
 import type { Sharp } from "sharp";
-import { disk } from "./storage";
-import { getAssetUrl } from "./storage";
+import { drive } from "./storage";
 
 const DEFAULT_THUMBNAIL_AREA = 230_400;
 
@@ -19,9 +18,9 @@ export interface Thumbnail {
 export async function uploadThumbnail(
   id: string,
   original: Sharp,
-  url: URL | string,
   thumbnailArea = DEFAULT_THUMBNAIL_AREA,
 ): Promise<Thumbnail> {
+  const disk = drive.use();
   const originalMetadata = await original.metadata();
   let width = originalMetadata.width!;
   let height = originalMetadata.height!;
@@ -55,7 +54,7 @@ export async function uploadThumbnail(
     throw error;
   }
   return {
-    thumbnailUrl: getAssetUrl(`media/${id}/thumbnail.webp`, url),
+    thumbnailUrl: await disk.getUrl(`media/${id}/thumbnail.webp`),
     thumbnailType: "image/webp",
     thumbnailWidth: thumbnailSize.width,
     thumbnailHeight: thumbnailSize.height,

src/oauth.test.ts

@@ -1,51 +1,55 @@
-import { describe, test } from "node:test";
+import { describe, it } from "node:test";
 import type { TestContext } from "node:test";
 
 import app from "./index";
 
-describe("OAuth", async () => {
-  await test("GET /.well-known/oauth-authorization-server", async (t: TestContext) => {
-    t.plan(10);
-
-    const response = await app.request(
-      "http://localhost:3000/.well-known/oauth-authorization-server",
-      {
-        method: "get",
-      },
-    );
-
-    t.assert.equal(response.status, 200, "Should return 200-ok");
-
-    const metadata = await response.json();
-
-    t.assert.equal(metadata.issuer, "http://localhost:3000/");
-    t.assert.equal(
-      metadata.authorization_endpoint,
-      "http://localhost:3000/oauth/authorize",
-    );
-    t.assert.equal(
-      metadata.token_endpoint,
-      "http://localhost:3000/oauth/token",
-    );
-    // Non-standard, mastodon extension:
-    t.assert.equal(
-      metadata.app_registration_endpoint,
-      "http://localhost:3000/api/v1/apps",
-    );
-
-    t.assert.deepStrictEqual(metadata.response_types_supported, ["code"]);
-    t.assert.deepStrictEqual(metadata.response_modes_supported, ["query"]);
-    t.assert.deepStrictEqual(metadata.grant_types_supported, [
-      "authorization_code",
-      "client_credentials",
-    ]);
-    t.assert.deepStrictEqual(metadata.token_endpoint_auth_methods_supported, [
-      "client_secret_post",
-    ]);
-
-    t.assert.ok(
-      Array.isArray(metadata.scopes_supported),
-      "Should return an array of scopes supported",
-    );
-  });
+describe("OAuth", () => {
+  it(
+    "Can GET /.well-known/oauth-authorization-server",
+    { plan: 10 },
+    async (t: TestContext) => {
+      // We use the full URL in this test as the route calculates values based
+      // on the Host header
+      const response = await app.request(
+        "http://localhost:3000/.well-known/oauth-authorization-server",
+        {
+          method: "GET",
+        },
+      );
+
+      t.assert.equal(response.status, 200, "Should return 200-ok");
+
+      const metadata = await response.json();
+
+      t.assert.equal(metadata.issuer, "http://localhost:3000/");
+      t.assert.equal(
+        metadata.authorization_endpoint,
+        "http://localhost:3000/oauth/authorize",
+      );
+      t.assert.equal(
+        metadata.token_endpoint,
+        "http://localhost:3000/oauth/token",
+      );
+      // Non-standard, mastodon extension:
+      t.assert.equal(
+        metadata.app_registration_endpoint,
+        "http://localhost:3000/api/v1/apps",
+      );
+
+      t.assert.deepStrictEqual(metadata.response_types_supported, ["code"]);
+      t.assert.deepStrictEqual(metadata.response_modes_supported, ["query"]);
+      t.assert.deepStrictEqual(metadata.grant_types_supported, [
+        "authorization_code",
+        "client_credentials",
+      ]);
+      t.assert.deepStrictEqual(metadata.token_endpoint_auth_methods_supported, [
+        "client_secret_post",
+      ]);
+
+      t.assert.ok(
+        Array.isArray(metadata.scopes_supported),
+        "Should return an array of scopes supported",
+      );
+    },
+  );
 });