WIP: tus routes; upload creation done; upload offset is getting updated in the db

Author: Rajat Saxena

apps/api/package.json

@@ -38,6 +38,8 @@
     "@medialit/models": "workspace:*",
     "@medialit/thumbnail": "workspace:*",
     "@medialit/utils": "workspace:^0.1.0",
+    "@tus/file-store": "^2.0.0",
+    "@tus/server": "^2.3.0",
     "aws-sdk": "^2.1692.0",
     "cors": "^2.8.5",
     "dotenv": "^16.4.7",

apps/api/src/apikey/middleware.ts

@@ -15,6 +15,7 @@ export default async function apikey(
     const reqKey = req.body.apikey;
 
     if (!reqKey) {
+        console.log("API key missing in request");
         return res.status(400).json({ error: BAD_REQUEST });
     }
 

apps/api/src/config/constants.ts

@@ -62,3 +62,11 @@ export const CDN_MAX_AGE = process.env.CDN_MAX_AGE
 
 export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT;
 export const HOSTNAME_OVERRIDE = process.env.HOSTNAME_OVERRIDE || ""; // Useful for hosting via Docker
+
+// Tus upload config
+export const TUS_UPLOAD_EXPIRATION_HOURS = parseInt(
+    process.env.TUS_UPLOAD_EXPIRATION_HOURS || "48",
+);
+export const TUS_CHUNK_SIZE = parseInt(
+    process.env.TUS_CHUNK_SIZE || "10485760",
+); // 10MB default

apps/api/src/index.ts

@@ -7,11 +7,13 @@ import passport from "passport";
 import mediaRoutes from "./media/routes";
 import presignedUrlRoutes from "./presigning/routes";
 import mediaSettingsRoutes from "./media-settings/routes";
+import tusRoutes from "./tus/routes";
 import logger from "./services/log";
 import { createUser, findByEmail } from "./user/queries";
 import { Apikey, Constants, User } from "@medialit/models";
 import { createApiKey } from "./apikey/queries";
 import { spawn } from "child_process";
+import { cleanupExpiredTusUploads } from "./tus/queries";
 
 connectToDatabase();
 const app = express();
@@ -29,7 +31,9 @@ app.get("/health", (req, res) => {
 
 app.use("/settings/media", mediaSettingsRoutes(passport));
 app.use("/media/presigned", presignedUrlRoutes);
+app.use("/media", tusRoutes);
 app.use("/media", mediaRoutes);
+// Mount TUS routes under /media so paths like /media/create/tus match correctly
 
 const port = process.env.PORT || 80;
 
@@ -41,6 +45,14 @@ checkDependencies().then(() => {
     app.listen(port, () => {
         logger.info(`Medialit server running at ${port}`);
     });
+
+    // Setup background cleanup job for expired tus uploads
+    setInterval(
+        async () => {
+            await cleanupExpiredTusUploads();
+        },
+        1000 * 60 * 60,
+    ); // Run every hour
 });
 
 async function checkDependencies() {

apps/api/src/presigning/handlers.ts

@@ -36,3 +36,30 @@ export async function getPresignedUrl(
         return res.status(500).json(err.message);
     }
 }
+
+export async function getPresignedTusUrl(
+    req: any,
+    res: any,
+    next: (...args: any[]) => void,
+) {
+    const { error } = validatePresigningOptions(req);
+    if (error) {
+        return res.status(400).json({ error: error.message });
+    }
+
+    try {
+        const presignedUrl = await preSignedUrlService.generateSignedUrlForTus({
+            userId: req.user.id,
+            apikey: req.apikey,
+            protocol: req.protocol,
+            host: HOSTNAME_OVERRIDE || req.get("Host"),
+            group: req.body.group,
+        });
+        // Extract signature from the URL
+        const signature = presignedUrl.split("signature=")[1];
+        return res.status(200).json({ signature });
+    } catch (err: any) {
+        logger.error({ err }, err.message);
+        return res.status(500).json(err.message);
+    }
+}

apps/api/src/presigning/routes.ts

@@ -1,8 +1,9 @@
 import express from "express";
 import apikey from "../apikey/middleware";
-import { getPresignedUrl } from "./handlers";
+import { getPresignedUrl, getPresignedTusUrl } from "./handlers";
 
 const router = express.Router();
 router.post("/create", apikey, getPresignedUrl);
+router.post("/tus/create", apikey, getPresignedTusUrl);
 
 export default router;

apps/api/src/presigning/service.ts

@@ -69,3 +69,26 @@ export async function cleanup(userId: string, signature: string) {
     await queries.deleteBySignature(signature);
     await queries.cleanupExpiredLinks(userId);
 }
+
+export async function generateSignedUrlForTus({
+    userId,
+    apikey,
+    protocol,
+    host,
+    group,
+}: GenerateSignedUrlProps): Promise<string> {
+    const presignedUrl = await queries.createPresignedUrl(
+        userId,
+        apikey,
+        group,
+    );
+
+    queries.cleanupExpiredLinks(userId).catch((err: any) => {
+        logger.error(
+            { err },
+            `Error while cleaning up expired links for ${userId}`,
+        );
+    });
+
+    return `${protocol}://${host}/media/create/tus?signature=${presignedUrl?.signature}`;
+}

apps/api/src/tus/finalize.ts

@@ -0,0 +1,203 @@
+import { readFileSync, createReadStream, rmdirSync, existsSync } from "fs";
+import path from "path";
+import thumbnail from "@medialit/thumbnail";
+import mongoose from "mongoose";
+import {
+    tempFileDirForUploads,
+    imagePattern,
+    imagePatternForThumbnailGeneration,
+    videoPattern,
+    USE_CLOUDFRONT,
+} from "../config/constants";
+import imageUtils from "@medialit/images";
+import {
+    foldersExist,
+    createFolders,
+} from "../media/utils/manage-files-on-disk";
+import type { MediaWithUserId } from "../media/model";
+import { putObject, UploadParams } from "../services/s3";
+import logger from "../services/log";
+import generateKey from "../media/utils/generate-key";
+import { getMediaSettings } from "../media-settings/queries";
+import generateFileName from "../media/utils/generate-file-name";
+import { createMedia } from "../media/queries";
+import getTags from "../media/utils/get-tags";
+import { getTusUpload, markTusUploadComplete } from "./queries";
+import * as presignedUrlService from "../presigning/service";
+
+const generateAndUploadThumbnail = async ({
+    workingDirectory,
+    key,
+    mimetype,
+    originalFilePath,
+    tags,
+}: {
+    workingDirectory: string;
+    key: string;
+    mimetype: string;
+    originalFilePath: string;
+    tags: string;
+}): Promise<boolean> => {
+    const thumbPath = `${workingDirectory}/thumb.webp`;
+
+    let isThumbGenerated = false;
+    if (imagePatternForThumbnailGeneration.test(mimetype)) {
+        await thumbnail.forImage(originalFilePath, thumbPath);
+        isThumbGenerated = true;
+    }
+    if (videoPattern.test(mimetype)) {
+        await thumbnail.forVideo(originalFilePath, thumbPath);
+        isThumbGenerated = true;
+    }
+
+    if (isThumbGenerated) {
+        await putObject({
+            Key: key,
+            Body: createReadStream(thumbPath),
+            ContentType: "image/webp",
+            ACL: USE_CLOUDFRONT ? "private" : "public-read",
+            Tagging: tags,
+        });
+    }
+
+    return isThumbGenerated;
+};
+
+export default async function finalizeUpload(uploadId: string) {
+    logger.info({ uploadId }, "Finalizing tus upload");
+
+    const tusUpload = await getTusUpload(uploadId);
+    if (!tusUpload) {
+        throw new Error(`Tus upload not found: ${uploadId}`);
+    }
+
+    if (tusUpload.isComplete) {
+        logger.info({ uploadId }, "Upload already finalized");
+        return;
+    }
+
+    const { userId, apikey, metadata, uploadLength, tempFilePath, signature } =
+        tusUpload;
+
+    // Read the completed file from tus data store
+    const tusFilePath = path.join(
+        `${tempFileDirForUploads}/tus-uploads`,
+        tempFilePath,
+    );
+
+    if (!existsSync(tusFilePath)) {
+        logger.error({ uploadId, tusFilePath }, "Tus file not found");
+        throw new Error(`Tus file not found: ${tusFilePath}`);
+    }
+
+    const mediaSettings = await getMediaSettings(userId, apikey);
+    const useWebP = mediaSettings?.useWebP || false;
+    const webpOutputQuality = mediaSettings?.webpOutputQuality || 0;
+
+    // Generate unique media ID
+    const fileName = generateFileName(metadata.filename);
+    const temporaryFolderForWork = `${tempFileDirForUploads}/${fileName.name}`;
+    if (!foldersExist([temporaryFolderForWork])) {
+        createFolders([temporaryFolderForWork]);
+    }
+
+    let fileExtension = path.extname(metadata.filename).replace(".", "");
+    let mimeType = metadata.mimetype;
+    if (useWebP && imagePattern.test(mimeType)) {
+        fileExtension = "webp";
+        mimeType = "image/webp";
+    }
+
+    const mainFilePath = `${temporaryFolderForWork}/main.${fileExtension}`;
+
+    // Copy file from tus store to working directory
+    const tusFileContent = readFileSync(tusFilePath);
+    require("fs").writeFileSync(mainFilePath, tusFileContent);
+
+    // Apply WebP conversion if needed
+    if (useWebP && imagePattern.test(metadata.mimetype)) {
+        await imageUtils.convertToWebp(mainFilePath, webpOutputQuality);
+    }
+
+    const uploadParams: UploadParams = {
+        Key: generateKey({
+            mediaId: fileName.name,
+            access: metadata.access === "public" ? "public" : "private",
+            filename: `main.${fileExtension}`,
+        }),
+        Body: createReadStream(mainFilePath),
+        ContentType: mimeType,
+        ACL: USE_CLOUDFRONT
+            ? "private"
+            : metadata.access === "public"
+              ? "public-read"
+              : "private",
+    };
+    const tags = getTags(userId, metadata.group);
+    uploadParams.Tagging = tags;
+
+    await putObject(uploadParams);
+
+    let isThumbGenerated = false;
+    try {
+        isThumbGenerated = await generateAndUploadThumbnail({
+            workingDirectory: temporaryFolderForWork,
+            mimetype: metadata.mimetype,
+            originalFilePath: mainFilePath,
+            key: generateKey({
+                mediaId: fileName.name,
+                access: "public",
+                filename: "thumb.webp",
+            }),
+            tags,
+        });
+    } catch (err: any) {
+        logger.error({ err }, err.message);
+    }
+
+    rmdirSync(temporaryFolderForWork, { recursive: true });
+
+    const mediaObject: MediaWithUserId = {
+        fileName: `main.${fileExtension}`,
+        mediaId: fileName.name,
+        userId: new mongoose.Types.ObjectId(userId),
+        apikey,
+        originalFileName: metadata.filename,
+        mimeType,
+        size: uploadLength,
+        thumbnailGenerated: isThumbGenerated,
+        caption: metadata.caption,
+        accessControl: metadata.access === "public" ? "public-read" : "private",
+        group: metadata.group,
+    };
+    const media = await createMedia(mediaObject);
+
+    // Mark upload as complete
+    await markTusUploadComplete(uploadId);
+
+    // Cleanup presigned URL if used
+    if (signature) {
+        presignedUrlService.cleanup(userId, signature).catch((err: any) => {
+            logger.error(
+                { err },
+                `Error in cleaning up expired links for ${userId}`,
+            );
+        });
+    }
+
+    // Cleanup tus file
+    try {
+        if (existsSync(tusFilePath)) {
+            require("fs").unlinkSync(tusFilePath);
+        }
+    } catch (err) {
+        logger.error({ err }, "Error cleaning up tus file");
+    }
+
+    logger.info(
+        { uploadId, mediaId: media.mediaId },
+        "Tus upload finalized successfully",
+    );
+
+    return media.mediaId;
+}

apps/api/src/tus/model.ts

@@ -0,0 +1,43 @@
+import type { Media } from "@medialit/models";
+import mongoose from "mongoose";
+
+export interface TusUpload {
+    uploadId: string;
+    userId: string;
+    apikey: string;
+    uploadLength: number;
+    uploadOffset: number;
+    metadata: Pick<
+        Media,
+        "fileName" | "mimeType" | "accessControl" | "caption" | "group"
+    >;
+    tempFilePath: string;
+    isComplete: boolean;
+    expiresAt?: Date;
+    signature?: string;
+}
+
+const TusUploadSchema = new mongoose.Schema<TusUpload>(
+    {
+        uploadId: { type: String, required: true, unique: true },
+        userId: { type: String, required: true },
+        apikey: { type: String, required: true },
+        uploadLength: { type: Number, required: true },
+        uploadOffset: { type: Number, required: true, default: 0 },
+        metadata: {
+            fileName: { type: String, required: true },
+            mimeType: { type: String, required: true },
+            accessControl: { type: String, required: true, default: "private" },
+            caption: String,
+            group: String,
+        },
+        signature: String,
+        tempFilePath: String,
+        isComplete: Boolean,
+        expiresAt: Date,
+    },
+    { timestamps: true },
+);
+
+export default mongoose.models.TusUpload ||
+    mongoose.model("TusUpload", TusUploadSchema);