Allow for filesystem upload

feature/media uploader

Author: nevo-david

.env.example

@@ -29,11 +29,14 @@ CLOUDFLARE_REGION="auto"
 #EMAIL_FROM_ADDRESS=""
 #EMAIL_FROM_NAME=""
 
+# Where will social media icons be saved - local or cloudflare.
+STORAGE_PROVIDER="local" 
+
 # Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
-#UPLOAD_DIRECTORY="/opt/postiz/uploads/"
+#UPLOAD_DIRECTORY=""
 
 # Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
-#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY="/opt/postiz/uploads/"
+#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY=""
 
 
 # Social Media API Settings

apps/backend/src/api/api.module.ts

@@ -46,18 +46,6 @@ const authenticatedController = [
 @Module({
   imports: [
     UploadModule,
-    ...(!!process.env.UPLOAD_DIRECTORY &&
-    !!process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY
-      ? [
-          ServeStaticModule.forRoot({
-            rootPath: process.env.UPLOAD_DIRECTORY,
-            serveRoot: '/' + process.env.NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY,
-            serveStaticOptions: {
-              index: false,
-            },
-          }),
-        ]
-      : []),
   ],
   controllers: [
     RootController,

apps/backend/src/api/routes/media.controller.ts

@@ -10,10 +10,13 @@ import handleR2Upload from '@gitroom/nestjs-libraries/upload/r2.uploader';
 import { FileInterceptor } from '@nestjs/platform-express';
 import { CustomFileValidationPipe } from '@gitroom/nestjs-libraries/upload/custom.upload.validation';
 import { SubscriptionService } from '@gitroom/nestjs-libraries/database/prisma/subscriptions/subscription.service';
+import { basename } from 'path';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @ApiTags('Media')
 @Controller('/media')
 export class MediaController {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _mediaService: MediaService,
     private _subscriptionService: SubscriptionService
@@ -33,19 +36,26 @@ export class MediaController {
     return {output: 'data:image/png;base64,' + await this._mediaService.generateImage(prompt, org)};
   }
 
-  @Post('/upload-simple')
+  @Post('/upload-server')
   @UseInterceptors(FileInterceptor('file'))
   @UsePipes(new CustomFileValidationPipe())
+  async uploadServer(
+    @GetOrgFromRequest() org: Organization,
+    @UploadedFile() file: Express.Multer.File
+  ) {
+    const uploadedFile = await this.storage.uploadFile(file);    
+    const filePath = uploadedFile.path.replace(process.env.UPLOAD_DIRECTORY, basename(process.env.UPLOAD_DIRECTORY));
+    return this._mediaService.saveFile(org.id, uploadedFile.originalname, filePath);
+  }
+
+  @Post('/upload-simple')
+  @UseInterceptors(FileInterceptor('file'))
   async uploadSimple(
     @GetOrgFromRequest() org: Organization,
-    @UploadedFile('file')
-    file: Express.Multer.File
+    @UploadedFile('file') file: Express.Multer.File
   ) {
-    const filePath =
-      file.path.indexOf('http') === 0
-        ? file.path
-        : file.path.replace(process.env.UPLOAD_DIRECTORY, '');
-    return this._mediaService.saveFile(org.id, file.originalname, filePath);
+    const getFile = await this.storage.uploadFile(file);
+    return this._mediaService.saveFile(org.id, getFile.originalname, getFile.path);
   }
 
   @Post('/:endpoint')

apps/frontend/next.config.js

@@ -15,12 +15,37 @@ const nextConfig = {
   transpilePackages: ['crypto-hash'],
   images: {
     remotePatterns: [
+      {
+        protocol: 'http',
+        hostname: '**',
+      },
       {
         protocol: 'https',
         hostname: '**',
       },
     ],
-  }
+  },
+  async redirects() {
+    return [
+      {
+        source: '/api/uploads/:path*',
+        destination:
+          process.env.STORAGE_PROVIDER === 'local' ? '/uploads/:path*' : '/404',
+        permanent: true,
+      },
+    ];
+  },
+  async rewrites() {
+    return [
+      {
+        source: '/uploads/:path*',
+        destination:
+          process.env.STORAGE_PROVIDER === 'local'
+            ? '/api/uploads/:path*'
+            : '/404',
+      },
+    ];
+  },
 };
 
 const plugins = [

apps/frontend/src/app/api/uploads/[[...path]]/route.ts

@@ -0,0 +1,47 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createReadStream, statSync } from 'fs';
+// @ts-ignore
+import mime from 'mime';
+
+async function* nodeStreamToIterator(stream: any) {
+  for await (const chunk of stream) {
+    yield chunk;
+  }
+}
+
+function iteratorToStream(iterator: any) {
+  return new ReadableStream({
+    async pull(controller) {
+      const { value, done } = await iterator.next();
+
+      if (done) {
+        controller.close();
+      } else {
+        controller.enqueue(new Uint8Array(value));
+      }
+    },
+  });
+}
+
+export const GET = (
+  request: NextRequest,
+  context: { params: { path: string[] } }
+) => {
+  const filePath =
+    process.env.UPLOAD_DIRECTORY + '/' + context.params.path.join('/');
+  const response = createReadStream(filePath);
+
+  const fileStats = statSync(filePath);
+  const contentType = mime.getType(filePath) || 'application/octet-stream';
+
+  const iterator = nodeStreamToIterator(response);
+  const webStream = iteratorToStream(iterator);
+  return new Response(webStream, {
+    headers: {
+      'Content-Type': contentType, // Set the appropriate content-type header
+      'Content-Length': fileStats.size.toString(), // Set the content-length header
+      'Last-Modified': fileStats.mtime.toUTCString(), // Set the last-modified header
+      'Cache-Control': 'public, max-age=31536000, immutable', // Example cache-control header
+    },
+  });
+};

apps/frontend/src/components/media/media.component.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { ChangeEvent, FC, useCallback, useEffect, useState } from 'react';
+import { FC, useCallback, useEffect, useState } from 'react';
 import { Button } from '@gitroom/react/form/button';
 import useSWR from 'swr';
 import { useFetch } from '@gitroom/helpers/utils/custom.fetch';
@@ -11,7 +11,6 @@ import EventEmitter from 'events';
 import { TopTitle } from '@gitroom/frontend/components/launches/helpers/top.title.component';
 import clsx from 'clsx';
 import { VideoFrame } from '@gitroom/react/helpers/video.frame';
-import { useToaster } from '@gitroom/react/toaster/toaster';
 import { LoadingComponent } from '@gitroom/frontend/components/layout/loading';
 import { MultipartFileUploader } from '@gitroom/frontend/components/media/new.uploader';
 import dynamic from 'next/dynamic';
@@ -186,6 +185,7 @@ export const MediaBox: FC<{
                   <img
                     className="w-full h-full object-cover"
                     src={mediaDirectory.set(media.path)}
+                    alt='media'
                   />
                 )}
               </div>

apps/frontend/src/components/media/new.uploader.tsx

@@ -2,34 +2,14 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react';
 // @ts-ignore
 import Uppy, { UploadResult } from '@uppy/core';
 // @ts-ignore
-import AwsS3Multipart from '@uppy/aws-s3-multipart';
-// @ts-ignore
 import { useFetch } from '@gitroom/helpers/utils/custom.fetch';
-
-import sha256 from 'sha256';
+import { getUppyUploadPlugin } from '@gitroom/react/helpers/uppy.upload'; 
 import { FileInput, ProgressBar } from '@uppy/react';
 
 // Uppy styles
 import '@uppy/core/dist/style.min.css';
 import '@uppy/dashboard/dist/style.min.css';
 
-const fetchUploadApiEndpoint = async (
-  fetch: any,
-  endpoint: string,
-  data: any
-) => {
-  const res = await fetch(`/media/${endpoint}`, {
-    method: 'POST',
-    body: JSON.stringify(data),
-    headers: {
-      accept: 'application/json',
-      'Content-Type': 'application/json',
-    },
-  });
-
-  return res.json();
-};
-
 export function MultipartFileUploader({
   onUploadSuccess,
   allowedFileTypes,
@@ -41,7 +21,7 @@ export function MultipartFileUploader({
   const [loaded, setLoaded] = useState(false);
   const [reload, setReload] = useState(false);
 
-  const onUploadSuccessExtended = useCallback((result: UploadResult) => {
+  const onUploadSuccessExtended = useCallback((result: UploadResult<any,any>) => {
     setReload(true);
     onUploadSuccess(result);
   }, [onUploadSuccess]);
@@ -78,7 +58,9 @@ export function MultipartFileUploaderAfter({
   onUploadSuccess: (result: UploadResult) => void;
   allowedFileTypes: string;
 }) {
+  const storageProvider = process.env.NEXT_PUBLIC_STORAGE_PROVIDER || "local";
   const fetch = useFetch();
+  
   const uppy = useMemo(() => {
     const uppy2 = new Uppy({
       autoProceed: true,
@@ -87,38 +69,17 @@ export function MultipartFileUploaderAfter({
         allowedFileTypes: allowedFileTypes.split(','),
         maxFileSize: 1000000000,
       },
-    }).use(AwsS3Multipart, {
-      // @ts-ignore
-      createMultipartUpload: async (file) => {
-        const arrayBuffer = await new Response(file.data).arrayBuffer();
-        // @ts-ignore
-        const fileHash = await sha256(arrayBuffer);
-        const contentType = file.type;
-        return fetchUploadApiEndpoint(fetch, 'create-multipart-upload', {
-          file,
-          fileHash,
-          contentType,
-        });
-      },
-      // @ts-ignore
-      listParts: (file, props) =>
-        fetchUploadApiEndpoint(fetch, 'list-parts', { file, ...props }),
-      // @ts-ignore
-      signPart: (file, props) =>
-        fetchUploadApiEndpoint(fetch, 'sign-part', { file, ...props }),
-      // @ts-ignore
-      abortMultipartUpload: (file, props) =>
-        fetchUploadApiEndpoint(fetch, 'abort-multipart-upload', {
-          file,
-          ...props,
-        }),
-      // @ts-ignore
-      completeMultipartUpload: (file, props) =>
-        fetchUploadApiEndpoint(fetch, 'complete-multipart-upload', {
-          file,
-          ...props,
-        }),
     });
+   
+    const { plugin, options } = getUppyUploadPlugin(storageProvider, fetch)
+    uppy2.use(plugin, options)
+    // Set additional metadata when a file is added
+    uppy2.on('file-added', (file) => {
+        uppy2.setFileMeta(file.id, {
+          useCloudflare: storageProvider === 'cloudflare' ? 'true' : 'false', // Example of adding a custom field
+          // Add more fields as needed
+        });
+      });
 
     uppy2.on('complete', (result) => {
       onUploadSuccess(result);
@@ -141,15 +102,17 @@ export function MultipartFileUploaderAfter({
 
   return (
     <>
+      {/* <Dashboard uppy={uppy} /> */}
       <ProgressBar uppy={uppy} />
       <FileInput
         uppy={uppy}
         locale={{
           strings: {
             chooseFiles: 'Upload',
           },
+          pluralize: (n) => n
         }}
-      />
+      /> 
     </>
   );
 }

apps/frontend/src/middleware.ts

@@ -7,7 +7,9 @@ import { getCookieUrlFromDomain } from '@gitroom/helpers/subdomain/subdomain.man
 export async function middleware(request: NextRequest) {
   const nextUrl = request.nextUrl;
   const authCookie = request.cookies.get('auth');
-
+  if (nextUrl.pathname.startsWith('/uploads/')) {
+    return NextResponse.next();
+  }
   // If the URL is logout, delete the cookie and redirect to login
   if (nextUrl.href.indexOf('/auth/logout') > -1) {
     const response = NextResponse.redirect(

libraries/nestjs-libraries/src/database/prisma/integrations/integration.repository.ts

@@ -3,12 +3,12 @@ import { Injectable } from '@nestjs/common';
 import dayjs from 'dayjs';
 import { Integration } from '@prisma/client';
 import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
-import { simpleUpload } from '@gitroom/nestjs-libraries/upload/r2.uploader';
-import axios from 'axios';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @Injectable()
 export class IntegrationRepository {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _integration: PrismaRepository<'integration'>,
     private _posts: PrismaRepository<'post'>
@@ -32,16 +32,10 @@ export class IntegrationRepository {
   async updateIntegration(id: string, params: Partial<Integration>) {
     if (
       params.picture &&
-      params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1
+      (params.picture.indexOf(process.env.CLOUDFLARE_BUCKET_URL!) === -1 ||
+        params.picture.indexOf(process.env.FRONTEND_URL!) === -1)
     ) {
-      const picture = await axios.get(params.picture, {
-        responseType: 'arraybuffer',
-      });
-      params.picture = await simpleUpload(
-        picture.data,
-        `${makeId(10)}.png`,
-        'image/png'
-      );
+      params.picture = await this.storage.uploadSimple(params.picture);
     }
 
     return this._integration.model.integration.update({

libraries/nestjs-libraries/src/database/prisma/integrations/integration.service.ts

@@ -21,9 +21,11 @@ import { timer } from '@gitroom/helpers/utils/timer';
 import { ioRedis } from '@gitroom/nestjs-libraries/redis/redis.service';
 import { RefreshToken } from '@gitroom/nestjs-libraries/integrations/social.abstract';
 import { IntegrationTimeDto } from '@gitroom/nestjs-libraries/dtos/integrations/integration.time.dto';
+import { UploadFactory } from '@gitroom/nestjs-libraries/upload/upload.factory';
 
 @Injectable()
 export class IntegrationService {
+  private storage = UploadFactory.createStorage();
   constructor(
     private _integrationRepository: IntegrationRepository,
     private _integrationManager: IntegrationManager,
@@ -50,13 +52,7 @@ export class IntegrationService {
     timezone?: number,
     customInstanceDetails?: string
   ) {
-    const loadImage = await axios.get(picture, { responseType: 'arraybuffer' });
-    const uploadedPicture = await simpleUpload(
-      loadImage.data,
-      `${makeId(10)}.png`,
-      'image/png'
-    );
-
+    const uploadedPicture = await this.storage.uploadSimple(picture);
     return this._integrationRepository.createOrUpdateIntegration(
       org,
       name,