Large file uploads via TUS

.changeset/good-books-double.md

@@ -0,0 +1,5 @@
+---
+"medialit": minor
+---
+
+API key and signature are passed via header instead of request body

.github/workflows/codeql.yml

@@ -0,0 +1,41 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: "51 18 * * 2"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ javascript ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+          queries: +security-and-quality
+
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v2
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:${{ matrix.language }}"

apps/api/__tests__/media/handlers.test.ts

@@ -80,6 +80,7 @@ describe("Media handlers", () => {
             },
             body: {},
             query: {},
+            headers: {},
         };
 
         const res = {
@@ -97,6 +98,7 @@ describe("Media handlers", () => {
         );
 
         const response = await uploadMedia(req, res, () => {});
+        console.log("Response", response);
         assert.strictEqual(response.code, 200);
     });
 });

apps/api/__tests__/media/storage-middleware.test.ts

@@ -7,6 +7,7 @@ import {
     maxStorageAllowedNotSubscribed,
     maxStorageAllowedSubscribed,
 } from "../../src/config/constants";
+import { NOT_ENOUGH_STORAGE } from "../../src/config/strings";
 
 describe("storageValidation middleware", () => {
     afterEach(() => {
@@ -108,11 +109,8 @@ describe("storageValidation middleware", () => {
         };
 
         const response = await storageValidation(req, res, next);
-        assert.strictEqual(response.code, 400);
-        assert.strictEqual(
-            response.data.error,
-            "You do not have enough storage space in your account to upload this file",
-        );
+        assert.strictEqual(response.code, 403);
+        assert.strictEqual(response.data.error, NOT_ENOUGH_STORAGE);
         assert.strictEqual(nextCalled, false);
     });
 
@@ -145,11 +143,8 @@ describe("storageValidation middleware", () => {
         };
 
         const response = await storageValidation(req, res, next);
-        assert.strictEqual(response.code, 400);
-        assert.strictEqual(
-            response.data.error,
-            "You do not have enough storage space in your account to upload this file",
-        );
+        assert.strictEqual(response.code, 403);
+        assert.strictEqual(response.data.error, NOT_ENOUGH_STORAGE);
         assert.strictEqual(nextCalled, false);
     });
 

apps/api/package.json

@@ -38,10 +38,12 @@
     "@medialit/models": "workspace:*",
     "@medialit/thumbnail": "workspace:*",
     "@medialit/utils": "workspace:^0.1.0",
+    "@tus/file-store": "^2.0.0",
+    "@tus/server": "^2.3.0",
     "aws-sdk": "^2.1692.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.7",
-    "express": "^4.18.2",
+    "express": "^4.2.0",
     "express-fileupload": "^1.3.1",
     "joi": "^17.6.0",
     "mongoose": "^8.0.1",

apps/api/src/apikey/middleware.ts

@@ -1,20 +1,18 @@
-import {
-    BAD_REQUEST,
-    SUBSCRIPTION_NOT_VALID,
-    UNAUTHORISED,
-} from "../config/strings";
+import { BAD_REQUEST, UNAUTHORISED } from "../config/strings";
 import { getApiKeyUsingKeyId } from "./queries";
 import { getUser } from "../user/queries";
 import { Apikey } from "@medialit/models";
+import logger from "../services/log";
 
 export default async function apikey(
     req: any,
     res: any,
     next: (...args: any[]) => void,
 ) {
-    const reqKey = req.body.apikey;
+    const reqKey = req.body?.apikey || req.headers["x-medialit-apikey"];
 
     if (!reqKey) {
+        logger.error({}, "API key is missing");
         return res.status(400).json({ error: BAD_REQUEST });
     }
 
@@ -23,13 +21,6 @@ export default async function apikey(
         return res.status(401).json({ error: UNAUTHORISED });
     }
 
-    // const isSubscriptionValid = await validateSubscription(
-    //     apiKey!.userId.toString(),
-    // );
-    // if (!isSubscriptionValid) {
-    //     return res.status(403).json({ error: SUBSCRIPTION_NOT_VALID });
-    // }
-
     req.user = await getUser(apiKey!.userId.toString());
     req.apikey = apiKey.key;
 

apps/api/src/config/constants.ts

@@ -6,19 +6,19 @@ export const tempFileDirForUploads = process.env.TEMP_FILE_DIR_FOR_UPLOADS;
 export const maxFileUploadSizeSubscribed = process.env
     .MAX_UPLOAD_SIZE_SUBSCRIBED
     ? +process.env.MAX_UPLOAD_SIZE_SUBSCRIBED
-    : 2147483648;
+    : 2147483648; // 2GB
 export const maxFileUploadSizeNotSubscribed = process.env
     .MAX_UPLOAD_SIZE_NOT_SUBSCRIBED
     ? +process.env.MAX_UPLOAD_SIZE_NOT_SUBSCRIBED
-    : 52428800;
+    : 52428800; // 50MB
 export const maxStorageAllowedSubscribed = process.env
     .MAX_STORAGE_ALLOWED_SUBSCRIBED
     ? +process.env.MAX_STORAGE_ALLOWED_SUBSCRIBED
-    : 107374182400;
+    : 107374182400; // 100GB
 export const maxStorageAllowedNotSubscribed = process.env
     .MAX_STORAGE_ALLOWED_NOT_SUBSCRIBED
     ? +process.env.MAX_STORAGE_ALLOWED_NOT_SUBSCRIBED
-    : 1073741824;
+    : 1073741824; // 1GB
 export const PRESIGNED_URL_VALIDITY_MINUTES = 5;
 export const PRESIGNED_URL_LENGTH = 100;
 export const MEDIA_ID_LENGTH = 40;
@@ -62,3 +62,8 @@ export const CDN_MAX_AGE = process.env.CDN_MAX_AGE
 
 export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT;
 export const HOSTNAME_OVERRIDE = process.env.HOSTNAME_OVERRIDE || ""; // Useful for hosting via Docker
+
+// Tus upload config
+export const TUS_UPLOAD_EXPIRATION_HOURS = parseInt(
+    process.env.TUS_UPLOAD_EXPIRATION_HOURS || "48",
+);

apps/api/src/config/strings.ts

@@ -6,3 +6,5 @@ export const FILE_IS_REQUIRED = "File is required";
 export const FILE_SIZE_EXCEEDED = "File size exceeded";
 export const NOT_FOUND = "Not found";
 export const PRESIGNED_URL_INVALID = "The link is invalid";
+export const NOT_ENOUGH_STORAGE =
+    "Not enough storage space in your account to upload this file";

apps/api/src/index.ts

@@ -5,13 +5,15 @@ import express from "express";
 import connectToDatabase from "./config/db";
 import passport from "passport";
 import mediaRoutes from "./media/routes";
-import presignedUrlRoutes from "./presigning/routes";
+import signatureRoutes from "./signature/routes";
 import mediaSettingsRoutes from "./media-settings/routes";
+import tusRoutes from "./tus/routes";
 import logger from "./services/log";
 import { createUser, findByEmail } from "./user/queries";
-import { Apikey, Constants, User } from "@medialit/models";
+import { Apikey, User } from "@medialit/models";
 import { createApiKey } from "./apikey/queries";
 import { spawn } from "child_process";
+import { Cleanup } from "./tus/cleanup";
 
 connectToDatabase();
 const app = express();
@@ -28,7 +30,8 @@ app.get("/health", (req, res) => {
 });
 
 app.use("/settings/media", mediaSettingsRoutes(passport));
-app.use("/media/presigned", presignedUrlRoutes);
+app.use("/media/signature", signatureRoutes);
+app.use("/media", tusRoutes);
 app.use("/media", mediaRoutes);
 
 const port = process.env.PORT || 80;
@@ -41,6 +44,14 @@ checkDependencies().then(() => {
     app.listen(port, () => {
         logger.info(`Medialit server running at ${port}`);
     });
+
+    // Setup background cleanup job for expired tus uploads
+    setInterval(
+        async () => {
+            await Cleanup();
+        },
+        1000 * 60 * 60, // 1 hours
+    );
 });
 
 async function checkDependencies() {

apps/api/src/media/handlers.ts

@@ -15,7 +15,8 @@ import logger from "../services/log";
 import { Request } from "express";
 import mediaService from "./service";
 import { getMediaCount as getCount, getTotalSpace } from "./queries";
-import { Constants, getSubscriptionStatus } from "@medialit/models";
+import { getSubscriptionStatus } from "@medialit/models";
+import { getSignatureFromReq } from "../signature/utils";
 
 function validateUploadOptions(req: Request): Joi.ValidationResult {
     const uploadSchema = Joi.object({
@@ -69,7 +70,7 @@ export async function uploadMedia(
             access,
             caption,
             group,
-            signature: req.query.signature,
+            signature: getSignatureFromReq(req),
         });
 
         const media = await mediaService.getMediaDetails({

apps/api/src/media/routes.ts

@@ -14,11 +14,13 @@ import {
     getMediaCount,
     getTotalSpaceOccupied,
 } from "./handlers";
-import presigned from "../presigning/middleware";
+import signatureMiddleware from "../signature/middleware";
 import storage from "./storage-middleware";
+import { getSignatureFromReq } from "../signature/utils";
 
 const router = express.Router();
 
+router.options("/create", cors());
 router.post(
     "/create",
     cors(),
@@ -31,9 +33,9 @@ router.post(
         },
     }),
     (req: Request, res: Response, next: (...args: any[]) => void) => {
-        const { signature } = req.query;
+        const signature = getSignatureFromReq(req);
         if (signature) {
-            presigned(
+            signatureMiddleware(
                 req as Request & { user: any; apikey: string },
                 res,
                 next,

apps/api/src/media/service.ts

@@ -35,7 +35,7 @@ import {
     getPaginatedMedia,
     createMedia,
 } from "./queries";
-import * as presignedUrlService from "../presigning/service";
+import * as presignedUrlService from "../signature/service";
 import getTags from "./utils/get-tags";
 import { getMainFileUrl, getThumbnailUrl } from "./utils/get-public-urls";
 

apps/api/src/media/storage-middleware.ts

@@ -1,7 +1,8 @@
 import { maxStorageAllowedNotSubscribed } from "../config/constants";
 import { maxStorageAllowedSubscribed } from "../config/constants";
-import { Constants, getSubscriptionStatus } from "@medialit/models";
+import { getSubscriptionStatus, User } from "@medialit/models";
 import mediaQueries from "./queries";
+import { NOT_ENOUGH_STORAGE } from "../config/strings";
 
 export default async function storageValidation(
     req: any,
@@ -14,21 +15,25 @@ export default async function storageValidation(
         });
     }
 
+    if (!(await hasEnoughStorage((req.files.file as any).size, req.user))) {
+        return res.status(403).json({
+            error: NOT_ENOUGH_STORAGE,
+        });
+    }
+
+    next();
+}
+
+export async function hasEnoughStorage(
+    size: number,
+    user: User,
+): Promise<boolean> {
     const totalSpaceOccupied = await mediaQueries.getTotalSpace({
-        userId: (req as any).user.id,
+        userId: user.id,
     });
-    const maxStorageAllowed = getSubscriptionStatus(req.user)
+    const maxStorageAllowed = getSubscriptionStatus(user)
         ? maxStorageAllowedSubscribed
         : maxStorageAllowedNotSubscribed;
 
-    if (
-        totalSpaceOccupied + (req.files?.file as any).size >
-        maxStorageAllowed
-    ) {
-        return res.status(400).json({
-            error: "You do not have enough storage space in your account to upload this file",
-        });
-    }
-
-    next();
+    return totalSpaceOccupied + size <= maxStorageAllowed;
 }

apps/api/src/presigning/handlers.ts → apps/api/src/signature/handlers.ts

@@ -12,7 +12,7 @@ function validatePresigningOptions(req: Request): Joi.ValidationResult {
     return uploadSchema.validate({ group });
 }
 
-export async function getPresignedUrl(
+export async function getSignature(
     req: any,
     res: any,
     next: (...args: any[]) => void,
@@ -23,14 +23,14 @@ export async function getPresignedUrl(
     }
 
     try {
-        const presignedUrl = await preSignedUrlService.generateSignedUrl({
+        const signature = await preSignedUrlService.generateSignature({
             userId: req.user.id,
             apikey: req.apikey,
             protocol: req.protocol,
             host: HOSTNAME_OVERRIDE || req.get("Host"),
             group: req.body.group,
         });
-        return res.status(200).json({ message: presignedUrl });
+        return res.status(200).json({ signature });
     } catch (err: any) {
         logger.error({ err }, err.message);
         return res.status(500).json(err.message);

apps/api/src/presigning/middleware.ts → apps/api/src/signature/middleware.ts

@@ -1,13 +1,14 @@
 import { Request, Response } from "express";
 import { PRESIGNED_URL_INVALID } from "../config/strings";
 import * as preSignedUrlService from "./service";
+import { getSignatureFromReq } from "./utils";
 
-export default async function presigned(
-    req: Request & { user: any; apikey: string },
+export default async function signature(
+    req: Request & { user?: any; apikey?: string },
     res: Response,
     next: (...args: any[]) => void,
 ) {
-    const { signature } = req.query;
+    const signature = getSignatureFromReq(req);
 
     const response = await preSignedUrlService.getUserAndGroupFromPresignedUrl(
         signature as string,

apps/api/src/presigning/queries.ts → apps/api/src/signature/queries.ts

@@ -1,12 +1,10 @@
 import mongoose from "mongoose";
 import PreSignedUrlModel, { PreSignedUrl } from "./model";
-import { getUniqueId } from "@medialit/utils";
-import { PRESIGNED_URL_LENGTH } from "../config/constants";
 
 export async function getPresignedUrl(
     signature: string,
 ): Promise<PreSignedUrl | null> {
     return await PreSignedUrlModel.findOne({ signature });
 }
 
 export async function deletePresignedUrl(
@@ -19,7 +17,7 @@
     userId: string,
     apikey: string,
     group?: string,
-): Promise<PreSignedUrl | undefined> {
+): Promise<PreSignedUrl> {
     const presignedUrl = await PreSignedUrlModel.create({
         userId,
         apikey,
@@ -36,5 +34,5 @@
 }
 
 export async function deleteBySignature(signature: string): Promise<void> {
     await PreSignedUrlModel.deleteOne({ signature });
 }