feat: fixes for uploading

Nevo David · Nevo David · commit a4f062c9c135 · 2024-10-06T19:41:22.000+07:00
diff --git a/apps/backend/src/api/api.module.ts b/apps/backend/src/api/api.module.ts
@@ -46,18 +46,6 @@ const authenticatedController = [
 @Module({
   imports: [
     UploadModule,
-    ...(!!process.env.UPLOAD_DIRECTORY &&
-    !!process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY
-      ? [
-          ServeStaticModule.forRoot({
-            rootPath: process.env.UPLOAD_DIRECTORY,
-            serveRoot: '/' + process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY,
-            serveStaticOptions: {
-              index: false,
-            },
-          }),
-        ]
-      : []),
   ],
   controllers: [
     RootController,
diff --git a/apps/backend/src/api/routes/media.controller.ts b/apps/backend/src/api/routes/media.controller.ts
@@ -50,17 +50,12 @@ export class MediaController {
 
   @Post('/upload-simple')
   @UseInterceptors(FileInterceptor('file'))
-  @UsePipes(new CustomFileValidationPipe())
   async uploadSimple(
     @GetOrgFromRequest() org: Organization,
-    @UploadedFile('file')
-    file: Express.Multer.File
+    @UploadedFile('file') file: Express.Multer.File
   ) {
-    const filePath =
-      file.path.indexOf('http') === 0
-        ? file.path
-        : file.path.replace(process.env.UPLOAD_DIRECTORY, '');
-    return this._mediaService.saveFile(org.id, file.originalname, filePath);
+    const getFile = await this.storage.uploadFile(file);
+    return this._mediaService.saveFile(org.id, getFile.originalname, getFile.path);
   }
 
   @Post('/:endpoint')
diff --git a/apps/frontend/next.config.js b/apps/frontend/next.config.js
@@ -15,12 +15,37 @@ const nextConfig = {
   transpilePackages: ['crypto-hash'],
   images: {
     remotePatterns: [
+      {
+        protocol: 'http',
+        hostname: '**',
+      },
       {
         protocol: 'https',
         hostname: '**',
       },
     ],
-  }
+  },
+  async redirects() {
+    return [
+      {
+        source: '/api/uploads/:path*',
+        destination:
+          process.env.STORAGE_PROVIDER === 'local' ? '/uploads/:path*' : '/404',
+        permanent: true,
+      },
+    ];
+  },
+  async rewrites() {
+    return [
+      {
+        source: '/uploads/:path*',
+        destination:
+          process.env.STORAGE_PROVIDER === 'local'
+            ? '/api/uploads/:path*'
+            : '/404',
+      },
+    ];
+  },
 };
 
 const plugins = [
diff --git a/apps/frontend/src/app/api/uploads/[[...path]]/route.ts b/apps/frontend/src/app/api/uploads/[[...path]]/route.ts
@@ -0,0 +1,47 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createReadStream, statSync } from 'fs';
+// @ts-ignore
+import mime from 'mime';
+
+async function* nodeStreamToIterator(stream: any) {
+  for await (const chunk of stream) {
+    yield chunk;
+  }
+}
+
+function iteratorToStream(iterator: any) {
+  return new ReadableStream({
+    async pull(controller) {
+      const { value, done } = await iterator.next();
+
+      if (done) {
+        controller.close();
+      } else {
+        controller.enqueue(new Uint8Array(value));
+      }
+    },
+  });
+}
+
+export const GET = (
+  request: NextRequest,
+  context: { params: { path: string[] } }
+) => {
+  const filePath =
+    process.env.UPLOAD_DIRECTORY + '/' + context.params.path.join('/');
+  const response = createReadStream(filePath);
+
+  const fileStats = statSync(filePath);
+  const contentType = mime.getType(filePath) || 'application/octet-stream';
+
+  const iterator = nodeStreamToIterator(response);
+  const webStream = iteratorToStream(iterator);
+  return new Response(webStream, {
+    headers: {
+      'Content-Type': contentType, // Set the appropriate content-type header
+      'Content-Length': fileStats.size.toString(), // Set the content-length header
+      'Last-Modified': fileStats.mtime.toUTCString(), // Set the last-modified header
+      'Cache-Control': 'public, max-age=31536000, immutable', // Example cache-control header
+    },
+  });
+};
diff --git a/apps/frontend/src/components/media/media.component.tsx b/apps/frontend/src/components/media/media.component.tsx
@@ -1,6 +1,6 @@
 'use client';
 
-import { ChangeEvent, FC, useCallback, useEffect, useState } from 'react';
+import { FC, useCallback, useEffect, useState } from 'react';
 import { Button } from '@gitroom/react/form/button';
 import useSWR from 'swr';
 import { useFetch } from '@gitroom/helpers/utils/custom.fetch';
@@ -11,7 +11,6 @@ import EventEmitter from 'events';
 import { TopTitle } from '@gitroom/frontend/components/launches/helpers/top.title.component';
 import clsx from 'clsx';
 import { VideoFrame } from '@gitroom/react/helpers/video.frame';
-import { useToaster } from '@gitroom/react/toaster/toaster';
 import { LoadingComponent } from '@gitroom/frontend/components/layout/loading';
 import { MultipartFileUploader } from '@gitroom/frontend/components/media/new.uploader';
 import dynamic from 'next/dynamic';
diff --git a/apps/frontend/src/middleware.ts b/apps/frontend/src/middleware.ts
@@ -7,7 +7,9 @@ import { getCookieUrlFromDomain } from '@gitroom/helpers/subdomain/subdomain.man
 export async function middleware(request: NextRequest) {
   const nextUrl = request.nextUrl;
   const authCookie = request.cookies.get('auth');
-
+  if (nextUrl.pathname.startsWith('/uploads/')) {
+    return NextResponse.next();
+  }
   // If the URL is logout, delete the cookie and redirect to login
   if (nextUrl.href.indexOf('/auth/logout') > -1) {
     const response = NextResponse.redirect(
diff --git a/libraries/nestjs-libraries/src/database/prisma/integrations/integration.repository.ts b/libraries/nestjs-libraries/src/database/prisma/integrations/integration.repository.ts
@@ -3,12 +3,12 @@ import { Injectable } from '@nestjs/common';
 import dayjs from 'dayjs';
 import { Integration } from '@prisma/client';
 import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
-import { simpleUpload } from '@gitroom/nestjs-libraries/upload/r2.uploader';
-import axios from 'axios';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @Injectable()
 export class IntegrationRepository {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _integration: PrismaRepository<'integration'>,
     private _posts: PrismaRepository<'post'>
@@ -32,16 +32,10 @@ export class IntegrationRepository {
   async updateIntegration(id: string, params: Partial<Integration>) {
     if (
       params.picture &&
-      params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1
+      (params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1 ||
+        params.picture.indexOf(process.env.FRONTEND_URL!) === -1)
     ) {
-      const picture = await axios.get(params.picture, {
-        responseType: 'arraybuffer',
-      });
-      params.picture = await simpleUpload(
-        picture.data,
-        `${makeId(10)}.png`,
-        'image/png'
-      );
+      params.picture = await this.storage.uploadSimple(params.picture);
     }
 
     return this._integration.model.integration.update({
diff --git a/libraries/nestjs-libraries/src/database/prisma/integrations/integration.service.ts b/libraries/nestjs-libraries/src/database/prisma/integrations/integration.service.ts
@@ -21,9 +21,11 @@ import { timer } from '@gitroom/helpers/utils/timer';
 import { ioRedis } from '@gitroom/nestjs-libraries/redis/redis.service';
 import { RefreshToken } from '@gitroom/nestjs-libraries/integrations/social.abstract';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @Injectable()
 export class IntegrationService {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _integrationRepository: IntegrationRepository,
     private _integrationManager: IntegrationManager,
@@ -50,13 +52,7 @@ export class IntegrationService {
     timezone?: number,
     customInstanceDetails?: string
   ) {
-    const loadImage = await axios.get(picture, { responseType: 'arraybuffer' });
-    const uploadedPicture = await simpleUpload(
-      loadImage.data,
-      `${makeId(10)}.png`,
-      'image/png'
-    );
-
+    const uploadedPicture = await this.storage.uploadSimple(picture);
     return this._integrationRepository.createOrUpdateIntegration(
       org,
       name,
diff --git a/libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts b/libraries/nestjs-libraries/src/integrations/social/bluesky.provider.ts
@@ -120,6 +120,8 @@ export class BlueskyProvider extends SocialAbstract implements SocialProvider {
     for (const post of postDetails) {
       const images = await Promise.all(
         post.media?.map(async (p) => {
+          const a = await fetch(p.url);
+          console.log(p.url);
           return await agent.uploadBlob(
             new Blob([
               await sharp(await (await fetch(p.url)).arrayBuffer())
diff --git a/libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts b/libraries/nestjs-libraries/src/integrations/social/reddit.provider.ts
@@ -143,7 +143,7 @@ export class RedditProvider extends SocialAbstract implements SocialProvider {
           ? {
               url: `${
                 firstPostSettings.value.media[0].path.indexOf('http') === -1
-                  ? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
+                  ? `${process.env.NEXT_PUBLIC_BACKEND_URL}/uploads`
                   : ``
               }${firstPostSettings.value.media[0].path}`,
             }
diff --git a/libraries/nestjs-libraries/src/upload/cloudflare.storage.ts b/libraries/nestjs-libraries/src/upload/cloudflare.storage.ts
@@ -1,8 +1,11 @@
 import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
 import 'multer';
-import {makeId} from "@gitroom/nestjs-libraries/services/make.is";
+import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
 import mime from 'mime-types';
+// @ts-ignore
+import {getExtension} from 'mime';
 import { IUploadProvider } from './upload.interface';
+import axios from 'axios';
 
 class CloudflareStorage implements IUploadProvider {
   private _client: S3Client;
@@ -13,7 +16,7 @@ class CloudflareStorage implements IUploadProvider {
     secretKey: string,
     private region: string,
     private _bucketName: string,
-    private _uploadUrl: string,
+    private _uploadUrl: string
   ) {
     this._client = new S3Client({
       endpoint: `https://${accountID}.r2.cloudflarestorage.com`,
@@ -25,10 +28,29 @@ class CloudflareStorage implements IUploadProvider {
     });
   }
 
+  async uploadSimple(path: string) {
+    const loadImage = await axios.get(path, { responseType: 'arraybuffer' });
+    const contentType = loadImage?.headers?.['content-type'] || loadImage?.headers?.['Content-Type'];
+    const extension = getExtension(contentType)!;
+    const id = makeId(10);
+
+    const params = {
+      Bucket: this._bucketName,
+      Key: `${id}.${extension}`,
+      Body: loadImage.data,
+      ContentType: contentType,
+    };
+
+    const command = new PutObjectCommand({ ...params });
+    await this._client.send(command);
+
+    return `${this._uploadUrl}/${id}.${extension}`;
+  }
+
   async uploadFile(file: Express.Multer.File): Promise<any> {
     const id = makeId(10);
     const extension = mime.extension(file.mimetype) || '';
-    
+
     // Create the PutObjectCommand to upload the file to Cloudflare R2
     const command = new PutObjectCommand({
       Bucket: this._bucketName,
@@ -56,12 +78,10 @@ class CloudflareStorage implements IUploadProvider {
   // Implement the removeFile method from IUploadProvider
   async removeFile(filePath: string): Promise<void> {
     // const fileName = filePath.split('/').pop(); // Extract the filename from the path
-
     // const command = new DeleteObjectCommand({
     //   Bucket: this._bucketName,
     //   Key: fileName,
     // });
-
     // await this._client.send(command);
   }
 }
diff --git a/libraries/nestjs-libraries/src/upload/local.storage.ts b/libraries/nestjs-libraries/src/upload/local.storage.ts
diff --git a/libraries/nestjs-libraries/src/upload/upload.interface.ts b/libraries/nestjs-libraries/src/upload/upload.interface.ts
diff --git a/libraries/react-shared-libraries/src/helpers/use.media.directory.ts b/libraries/react-shared-libraries/src/helpers/use.media.directory.ts
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json

Original file line number	Diff line number	Diff line change
`@@ -143,7 +143,7 @@ export class RedditProvider extends SocialAbstract implements SocialProvider {`
`143`	`143`	`? {`
`144`	`144`	url: `${
`145`	`145`	`firstPostSettings.value.media[0].path.indexOf('http') === -1`
`146`		- ? `${process.env.NEXT_PUBLIC_BACKEND_URL}/${process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY}`
	`146`	+ ? `${process.env.NEXT_PUBLIC_BACKEND_URL}/uploads`
`147`	`147`	: ``
`148`	`148`	}${firstPostSettings.value.media[0].path}`,
`149`	`149`	`}`