feat: Comprehensive storage provider system (S3-compatible, FTP/SFTP)

.env.example

@@ -30,14 +30,82 @@ CLOUDFLARE_REGION="auto"
 #EMAIL_FROM_NAME=""
 #DISABLE_REGISTRATION=false
 
-# Where will social media icons be saved - local or cloudflare.
+# ===================================================================
+# STORAGE CONFIGURATION
+# ===================================================================
+# Where will social media icons and uploaded files be saved.
+# Supported providers: local, cloudflare, s3-compatible, ftp, sftp
 STORAGE_PROVIDER="local"
 
-# Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
-#UPLOAD_DIRECTORY=""
-
-# Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
-#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY=""
+# === LOCAL STORAGE ===
+# Your upload directory path if you host your files locally
+#UPLOAD_DIRECTORY="/path/to/upload/directory"
+#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY="/uploads"
+
+# === CLOUDFLARE R2 STORAGE ===
+# Cloudflare R2 is S3-compatible and used for the existing cloudflare provider
+#CLOUDFLARE_ACCOUNT_ID="your-account-id"
+#CLOUDFLARE_ACCESS_KEY="your-access-key"
+#CLOUDFLARE_SECRET_ACCESS_KEY="your-secret-access-key"
+#CLOUDFLARE_BUCKETNAME="your-bucket-name"
+#CLOUDFLARE_BUCKET_URL="https://your-bucket-url.r2.cloudflarestorage.com/"
+#CLOUDFLARE_REGION="auto"
+
+# === S3-COMPATIBLE STORAGE ===
+# Works with AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, Wasabi, and more
+#S3_COMPATIBLE_ACCESS_KEY="your-access-key"
+#S3_COMPATIBLE_SECRET_KEY="your-secret-key"
+#S3_COMPATIBLE_REGION="us-east-1"
+#S3_COMPATIBLE_BUCKET="your-bucket-name"
+
+# Optional: Custom S3 endpoint (for non-AWS providers)
+#S3_COMPATIBLE_ENDPOINT="https://s3.us-east-1.amazonaws.com"
+# Examples:
+# MinIO: "http://localhost:9000" or "https://minio.example.com"
+# DigitalOcean: "https://nyc3.digitaloceanspaces.com"
+# Backblaze B2: "https://s3.us-west-001.backblazeb2.com"
+# Wasabi: "https://s3.us-east-1.wasabisys.com"
+
+# Optional: Public URL where files will be accessible (overrides auto-generated URL)
+#S3_COMPATIBLE_PUBLIC_URL="https://cdn.example.com"
+# Examples:
+# AWS S3: "https://my-bucket.s3.us-east-1.amazonaws.com"
+# MinIO: "https://cdn.example.com" (with reverse proxy)
+# DigitalOcean: "https://my-space.nyc3.cdn.digitaloceanspaces.com"
+
+# Optional: Use path-style URLs (required for MinIO, optional for others)
+#S3_COMPATIBLE_PATH_STYLE=false
+
+# Optional: Generate signed URLs for private buckets
+#S3_COMPATIBLE_SIGNED_URLS=false
+#S3_COMPATIBLE_SIGNED_URL_EXPIRY=3600
+
+# === FTP STORAGE ===
+# Upload files via FTP protocol to any FTP server
+#FTP_HOST="ftp.example.com"
+#FTP_PORT=21
+#FTP_USER="ftpuser"
+#FTP_PASSWORD="ftppassword"
+#FTP_REMOTE_PATH="/public_html/uploads"
+#FTP_PUBLIC_URL="https://example.com/uploads"
+
+# Optional: Use FTPS (FTP over SSL/TLS)
+#FTP_SECURE=false
+
+# Optional: Use passive FTP mode (recommended for most setups)
+#FTP_PASSIVE_MODE=true
+
+# === SFTP STORAGE ===
+# Upload files via SFTP (SSH File Transfer Protocol)
+#SFTP_HOST="sftp.example.com"
+#SFTP_PORT=22
+#SFTP_USER="sftpuser"
+#SFTP_REMOTE_PATH="/var/www/uploads"
+#SFTP_PUBLIC_URL="https://example.com/uploads"
+
+# Authentication: Use either password OR SSH key (not both)
+#SFTP_PASSWORD="sftppassword"
+#SFTP_PRIVATE_KEY_PATH="/path/to/private/key"
 
 # Social Media API Settings
 X_API_KEY=""

.gitignore

@@ -39,6 +39,9 @@ testem.log
 .DS_Store
 Thumbs.db
 
+# macOS resource fork files
+._*
+
 .nx/cache
 .nx/workspace-data
 

libraries/nestjs-libraries/src/upload/ftp.storage.ts

@@ -0,0 +1,243 @@
+import * as ftp from 'basic-ftp';
+import 'multer';
+import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
+import mime from 'mime-types';
+// @ts-ignore
+import { getExtension } from 'mime';
+import { IUploadProvider } from './upload.interface';
+import axios from 'axios';
+import { Readable } from 'stream';
+
+/**
+ * FTP storage provider for uploading files via FTP protocol
+ * Supports both FTP and FTPS (FTP over SSL/TLS)
+ */
+export class FTPStorage implements IUploadProvider {
+  private _host: string;
+  private _port: number;
+  private _user: string;
+  private _password: string;
+  private _remotePath: string;
+  private _publicUrl: string;
+  private _secure: boolean;
+  private _passiveMode: boolean;
+
+  /**
+   * Initialize FTP storage provider
+   * @param host - FTP server hostname
+   * @param port - FTP server port (default: 21)
+   * @param user - FTP username
+   * @param password - FTP password
+   * @param remotePath - Remote directory path on FTP server
+   * @param publicUrl - Public URL where uploaded files will be accessible via HTTP
+   * @param secure - Use FTPS (FTP over SSL/TLS) (default: false)
+   * @param passiveMode - Use passive FTP mode (default: true)
+   */
+  constructor(
+    host: string,
+    port: number = 21,
+    user: string,
+    password: string,
+    remotePath: string,
+    publicUrl: string,
+    secure: boolean = false,
+    passiveMode: boolean = true
+  ) {
+    this._host = host;
+    this._port = port;
+    this._user = user;
+    this._password = password;
+    this._remotePath = remotePath.endsWith('/') ? remotePath.slice(0, -1) : remotePath;
+    this._publicUrl = publicUrl.endsWith('/') ? publicUrl.slice(0, -1) : publicUrl;
+    this._secure = secure;
+    this._passiveMode = passiveMode;
+  }
+
+  /**
+   * Create and configure FTP client connection
+   * @returns Promise<ftp.Client> - Configured FTP client
+   */
+  private async createFTPClient(): Promise<ftp.Client> {
+    const client = new ftp.Client();
+
+    try {
+      // Configure connection timeout and keep-alive
+      client.ftp.timeout = 30000; // 30 seconds timeout
+
+      await client.access({
+        host: this._host,
+        port: this._port,
+        user: this._user,
+        password: this._password,
+        secure: this._secure,
+        secureOptions: this._secure ? { rejectUnauthorized: false } : undefined,
+      });
+
+      // Set passive mode
+      client.ftp.pasv = this._passiveMode;
+
+      return client;
+    } catch (error) {
+      client.close();
+      throw new Error(`Failed to connect to FTP server: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  /**
+   * Generate date-based directory structure (YYYY/MM/DD)
+   * @returns string - Directory path
+   */
+  private generateDatePath(): string {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    const day = String(now.getDate()).padStart(2, '0');
+    return `${year}/${month}/${day}`;
+  }
+
+  /**
+   * Ensure directory exists on FTP server, creating if necessary
+   * @param client - FTP client
+   * @param dirPath - Directory path to create
+   */
+  private async ensureDirectory(client: ftp.Client, dirPath: string): Promise<void> {
+    try {
+      await client.ensureDir(dirPath);
+    } catch (error) {
+      throw new Error(`Failed to create directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  /**
+   * Upload a file from a URL via FTP
+   * @param path - URL of the file to upload
+   * @returns Promise<string> - The public URL where the uploaded file can be accessed
+   */
+  async uploadSimple(path: string): Promise<string> {
+    let client: ftp.Client | null = null;
+
+    try {
+      // Download file from URL
+      const response = await axios.get(path, { responseType: 'arraybuffer' });
+      const contentType = response.headers['content-type'] || 'application/octet-stream';
+
+      const extension = getExtension(contentType);
+      if (!extension) {
+        throw new Error(`Unable to determine file extension for content type: ${contentType}`);
+      }
+
+      // Generate file path
+      const datePath = this.generateDatePath();
+      const id = makeId(10);
+      const fileName = `${id}.${extension}`;
+      const remoteDir = `${this._remotePath}/${datePath}`;
+      const remoteFilePath = `${remoteDir}/${fileName}`;
+      const publicPath = `${datePath}/${fileName}`;
+
+      // Connect to FTP server
+      client = await this.createFTPClient();
+
+      // Ensure directory exists
+      await this.ensureDirectory(client, remoteDir);
+
+      // Create readable stream from buffer
+      const stream = Readable.from(response.data);
+
+      // Upload file
+      await client.uploadFrom(stream, remoteFilePath);
+
+      return `${this._publicUrl}/${publicPath}`;
+    } catch (error) {
+      console.error('Error uploading file via FTP uploadSimple:', error);
+      throw new Error(`Failed to upload file from ${path}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    } finally {
+      if (client) {
+        client.close();
+      }
+    }
+  }
+
+  /**
+   * Upload a file from form data via FTP
+   * @param file - The uploaded file object
+   * @returns Promise<any> - File information including the public access URL
+   */
+  async uploadFile(file: Express.Multer.File): Promise<any> {
+    let client: ftp.Client | null = null;
+
+    try {
+      // Generate file path
+      const datePath = this.generateDatePath();
+      const id = makeId(10);
+      const extension = mime.extension(file.mimetype) || 'bin';
+      const fileName = `${id}.${extension}`;
+      const remoteDir = `${this._remotePath}/${datePath}`;
+      const remoteFilePath = `${remoteDir}/${fileName}`;
+      const publicPath = `${datePath}/${fileName}`;
+
+      // Connect to FTP server
+      client = await this.createFTPClient();
+
+      // Ensure directory exists
+      await this.ensureDirectory(client, remoteDir);
+
+      // Create readable stream from buffer
+      const stream = Readable.from(file.buffer);
+
+      // Upload file
+      await client.uploadFrom(stream, remoteFilePath);
+
+      const publicUrl = `${this._publicUrl}/${publicPath}`;
+
+      return {
+        filename: fileName,
+        mimetype: file.mimetype,
+        size: file.size,
+        buffer: file.buffer,
+        originalname: fileName,
+        fieldname: 'file',
+        path: publicUrl,
+        destination: publicUrl,
+        encoding: '7bit',
+        stream: file.buffer as any,
+      };
+    } catch (error) {
+      console.error('Error uploading file via FTP uploadFile:', error);
+      throw new Error(`Failed to upload file ${file.originalname}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    } finally {
+      if (client) {
+        client.close();
+      }
+    }
+  }
+
+  /**
+   * Remove a file from FTP storage
+   * @param filePath - The public URL of the file to remove
+   * @returns Promise<void>
+   */
+  async removeFile(filePath: string): Promise<void> {
+    let client: ftp.Client | null = null;
+
+    try {
+      // Extract relative path from public URL
+      const relativePath = filePath.replace(this._publicUrl + '/', '');
+      const remoteFilePath = `${this._remotePath}/${relativePath}`;
+
+      // Connect to FTP server
+      client = await this.createFTPClient();
+
+      // Remove file
+      await client.remove(remoteFilePath);
+    } catch (error) {
+      console.error('Error removing file via FTP:', error);
+      // Don't throw error for file removal failures to avoid breaking the application
+      // Just log the error as the file might already be deleted or not exist
+      console.warn(`Failed to remove file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    } finally {
+      if (client) {
+        client.close();
+      }
+    }
+  }
+}