Large file uploads fixes

apps/api/src/config/constants.ts

@@ -19,8 +19,10 @@ export const maxStorageAllowedNotSubscribed = process.env
     .MAX_STORAGE_ALLOWED_NOT_SUBSCRIBED
     ? +process.env.MAX_STORAGE_ALLOWED_NOT_SUBSCRIBED
     : 1073741824; // 1GB
-export const PRESIGNED_URL_VALIDITY_MINUTES = 5;
-export const PRESIGNED_URL_LENGTH = 100;
+export const SIGNATURE_VALIDITY_MINUTES = +(
+    process.env.SIGNATURE_VALIDITY_MINUTES || "1440"
+); // 1 day
+export const SIGNATURE_LENGTH = 100;
 export const MEDIA_ID_LENGTH = 40;
 export const APIKEY_RESTRICTION_REFERRER = "referrer";
 export const APIKEY_RESTRICTION_IP = "ipaddress";
@@ -62,8 +64,3 @@ export const CDN_MAX_AGE = process.env.CDN_MAX_AGE
 
 export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT;
 export const HOSTNAME_OVERRIDE = process.env.HOSTNAME_OVERRIDE || ""; // Useful for hosting via Docker
-
-// Tus upload config
-export const TUS_UPLOAD_EXPIRATION_HOURS = parseInt(
-    process.env.TUS_UPLOAD_EXPIRATION_HOURS || "48",
-);

apps/api/src/media/handlers.ts

@@ -1,7 +1,5 @@
 import Joi from "joi";
 import {
-    maxFileUploadSizeNotSubscribed,
-    maxFileUploadSizeSubscribed,
     maxStorageAllowedNotSubscribed,
     maxStorageAllowedSubscribed,
 } from "../config/constants";
@@ -17,6 +15,7 @@ import mediaService from "./service";
 import { getMediaCount as getCount, getTotalSpace } from "./queries";
 import { getSubscriptionStatus } from "@medialit/models";
 import { getSignatureFromReq } from "../signature/utils";
+import getMaxFileUploadSize from "./utils/get-max-file-upload-size";
 
 function validateUploadOptions(req: Request): Joi.ValidationResult {
     const uploadSchema = Joi.object({
@@ -28,12 +27,6 @@ function validateUploadOptions(req: Request): Joi.ValidationResult {
     return uploadSchema.validate({ caption, access, group });
 }
 
-function getMaxFileUploadSize(req: any): number {
-    return getSubscriptionStatus(req.user)
-        ? maxFileUploadSizeSubscribed
-        : maxFileUploadSizeNotSubscribed;
-}
-
 export async function uploadMedia(
     req: any,
     res: any,

apps/api/src/media/queries.ts

@@ -1,3 +1,4 @@
+import mongoose from "mongoose";
 import { numberOfRecordsPerPage } from "../config/constants";
 import GetPageProps from "./GetPageProps";
 import MediaModel, { MediaWithUserId } from "./model";
@@ -32,31 +33,27 @@ export async function getTotalSpace({
     userId,
     apikey,
 }: {
-    userId: string;
+    userId: mongoose.Types.ObjectId;
     apikey?: string;
 }): Promise<number> {
-    const result = await MediaModel
-        // calculate sum of size of all media files
-        .aggregate([
-            {
-                $match: {
-                    userId,
-                    apikey,
-                },
-            },
-            {
-                $group: {
-                    _id: null,
-                    totalSize: { $sum: "$size" },
-                },
+    const query = apikey ? { userId, apikey } : { userId };
+    const result = await MediaModel.aggregate([
+        {
+            $match: query,
+        },
+        {
+            $group: {
+                _id: null,
+                totalSize: { $sum: "$size" },
             },
-            {
-                $project: {
-                    _id: 0,
-                    totalSize: 1,
-                },
+        },
+        {
+            $project: {
+                _id: 0,
+                totalSize: 1,
             },
-        ]);
+        },
+    ]);
 
     if (result.length === 0) {
         return 0;

apps/api/src/media/storage-middleware.ts

@@ -3,6 +3,7 @@ import { maxStorageAllowedSubscribed } from "../config/constants";
 import { getSubscriptionStatus, User } from "@medialit/models";
 import mediaQueries from "./queries";
 import { NOT_ENOUGH_STORAGE } from "../config/strings";
+import mongoose from "mongoose";
 
 export default async function storageValidation(
     req: any,
@@ -26,10 +27,10 @@ export default async function storageValidation(
 
 export async function hasEnoughStorage(
     size: number,
-    user: User,
+    user: User & { _id: mongoose.Types.ObjectId },
 ): Promise<boolean> {
     const totalSpaceOccupied = await mediaQueries.getTotalSpace({
-        userId: user.id,
+        userId: user._id,
     });
     const maxStorageAllowed = getSubscriptionStatus(user)
         ? maxStorageAllowedSubscribed

apps/api/src/media/utils/get-max-file-upload-size.ts

@@ -0,0 +1,11 @@
+import {
+    maxFileUploadSizeNotSubscribed,
+    maxFileUploadSizeSubscribed,
+} from "../../config/constants";
+import { getSubscriptionStatus } from "@medialit/models";
+
+export default function getMaxFileUploadSize(req: any): number {
+    return getSubscriptionStatus(req.user)
+        ? maxFileUploadSizeSubscribed
+        : maxFileUploadSizeNotSubscribed;
+}

apps/api/src/signature/model.ts

@@ -1,7 +1,7 @@
 import mongoose from "mongoose";
 import {
-    PRESIGNED_URL_LENGTH,
-    PRESIGNED_URL_VALIDITY_MINUTES,
+    SIGNATURE_LENGTH,
+    SIGNATURE_VALIDITY_MINUTES,
 } from "../config/constants";
 import { getUniqueId } from "@medialit/utils";
 
@@ -21,15 +21,14 @@ const PreSignedUrlSchema = new mongoose.Schema<PreSignedUrl>(
         signature: {
             type: String,
             required: true,
-            default: () => getUniqueId(PRESIGNED_URL_LENGTH),
+            default: () => getUniqueId(SIGNATURE_LENGTH),
         },
         validTill: {
             type: Date,
             required: true,
             default: () =>
                 new Date(
-                    new Date().getTime() +
-                        PRESIGNED_URL_VALIDITY_MINUTES * 60000,
+                    new Date().getTime() + SIGNATURE_VALIDITY_MINUTES * 60000,
                 ),
         },
         group: String,

apps/api/src/tus/finalize.ts

@@ -1,4 +1,9 @@
-import { readFileSync, createReadStream, rmdirSync, existsSync } from "fs";
+import {
+    createReadStream,
+    existsSync,
+    copyFileSync,
+    promises as fsPromises,
+} from "fs";
 import path from "path";
 import thumbnail from "@medialit/thumbnail";
 import mongoose from "mongoose";
@@ -79,9 +84,7 @@ export default async function finalizeUpload(uploadId: string) {
 
     const mainFilePath = `${temporaryFolderForWork}/main.${fileExtension}`;
 
-    //Copy file from tus store to working directory
-    const tusFileContent = readFileSync(tusFilePath);
-    require("fs").writeFileSync(mainFilePath, tusFileContent);
+    copyFileSync(tusFilePath, mainFilePath);
 
     // Apply WebP conversion if needed
     if (useWebP && imagePattern.test(metadata.mimeType)) {
@@ -124,7 +127,7 @@ export default async function finalizeUpload(uploadId: string) {
         logger.error({ err }, err.message);
     }
 
-    rmdirSync(temporaryFolderForWork, { recursive: true });
+    await fsPromises.rm(temporaryFolderForWork, { recursive: true });
 
     const mediaObject: MediaWithUserId = {
         fileName: `main.${fileExtension}`,
@@ -181,26 +184,27 @@ const generateAndUploadThumbnail = async ({
     tags: string;
 }): Promise<boolean> => {
     const thumbPath = `${workingDirectory}/thumb.webp`;
+    let isGenerated = false;
 
-    let isThumbGenerated = false;
     if (imagePatternForThumbnailGeneration.test(mimetype)) {
         await thumbnail.forImage(originalFilePath, thumbPath);
-        isThumbGenerated = true;
+        isGenerated = true;
     }
     if (videoPattern.test(mimetype)) {
         await thumbnail.forVideo(originalFilePath, thumbPath);
-        isThumbGenerated = true;
+        isGenerated = true;
     }
 
-    if (isThumbGenerated) {
+    if (isGenerated) {
         await putObject({
             Key: key,
             Body: createReadStream(thumbPath),
             ContentType: "image/webp",
             ACL: USE_CLOUDFRONT ? "private" : "public-read",
             Tagging: tags,
         });
+        await fsPromises.rm(thumbPath);
     }
 
-    return isThumbGenerated;
+    return isGenerated;
 };

apps/api/src/tus/queries.ts

@@ -1,14 +1,14 @@
-import { TUS_UPLOAD_EXPIRATION_HOURS } from "../config/constants";
+import { SIGNATURE_VALIDITY_MINUTES } from "../config/constants";
 import TusUploadModel, { TusUpload } from "./model";
 
 type TusUploadDocument = any;
 
 export async function createTusUpload(
     data: Omit<TusUpload, "uploadOffset" | "isComplete">,
 ): Promise<TusUploadDocument> {
-    // const uploadId = new mongoose.Types.ObjectId().toString();
     const expiresAt = new Date();
-    expiresAt.setHours(expiresAt.getHours() + TUS_UPLOAD_EXPIRATION_HOURS);
+    const signatureValidityHours = SIGNATURE_VALIDITY_MINUTES / 60;
+    expiresAt.setHours(expiresAt.getHours() + signatureValidityHours);
 
     const tusUploadData: TusUpload = {
         uploadId: data.uploadId,

apps/api/src/tus/tus-server.ts

@@ -5,6 +5,7 @@ import logger from "../services/log";
 import finalizeUpload from "./finalize";
 import * as preSignedUrlService from "../signature/service";
 import {
+    FILE_SIZE_EXCEEDED,
     NOT_ENOUGH_STORAGE,
     PRESIGNED_URL_INVALID,
     UNAUTHORISED,
@@ -14,6 +15,7 @@ import { getApiKeyUsingKeyId } from "../apikey/queries";
 import { getUser } from "../user/queries";
 import { hasEnoughStorage } from "../media/storage-middleware";
 import { createTusUpload, updateTusUploadOffset } from "./queries";
+import getMaxFileUploadSize from "../media/utils/get-max-file-upload-size";
 
 const store = new FileStore({
     directory: `${tempFileDirForUploads}/tus-uploads`,
@@ -41,6 +43,13 @@ export const server = new Server({
         const { user, apikey } = req;
 
         try {
+            const allowedFileSize = getMaxFileUploadSize(req);
+            if (upload.size > allowedFileSize) {
+                throw {
+                    status_code: 403,
+                    body: `${FILE_SIZE_EXCEEDED}. Allowed: ${allowedFileSize} bytes`,
+                };
+            }
             if (!(await hasEnoughStorage(upload.size, user))) {
                 throw {
                     status_code: 403,

apps/api/src/user/queries.ts

@@ -1,7 +1,10 @@
 import { SubscriptionStatus, User } from "@medialit/models";
 import UserModel from "./model";
+import mongoose from "mongoose";
 
-export async function getUser(id: string): Promise<User | null> {
+export async function getUser(
+    id: string,
+): Promise<(User & { _id: mongoose.Types.ObjectId }) | null> {
     return UserModel.findById(id);
 }