diff --git a/.env.example b/.env.example index 61d2a0206..5d42baea4 100644 --- a/.env.example +++ b/.env.example @@ -30,14 +30,82 @@ CLOUDFLARE_REGION="auto" #EMAIL_FROM_NAME="" #DISABLE_REGISTRATION=false -# Where will social media icons be saved - local or cloudflare. +# =================================================================== +# STORAGE CONFIGURATION +# =================================================================== +# Where will social media icons and uploaded files be saved. +# Supported providers: local, cloudflare, s3-compatible, ftp, sftp STORAGE_PROVIDER="local" -# Your upload directory path if you host your files locally, otherwise Cloudflare will be used. -#UPLOAD_DIRECTORY="" - -# Your upload directory path if you host your files locally, otherwise Cloudflare will be used. -#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY="" +# === LOCAL STORAGE === +# Your upload directory path if you host your files locally +#UPLOAD_DIRECTORY="/path/to/upload/directory" +#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY="/uploads" + +# === CLOUDFLARE R2 STORAGE === +# Cloudflare R2 is S3-compatible and used for the existing cloudflare provider +#CLOUDFLARE_ACCOUNT_ID="your-account-id" +#CLOUDFLARE_ACCESS_KEY="your-access-key" +#CLOUDFLARE_SECRET_ACCESS_KEY="your-secret-access-key" +#CLOUDFLARE_BUCKETNAME="your-bucket-name" +#CLOUDFLARE_BUCKET_URL="https://your-bucket-url.r2.cloudflarestorage.com/" +#CLOUDFLARE_REGION="auto" + +# === S3-COMPATIBLE STORAGE === +# Works with AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, Wasabi, and more +#S3_COMPATIBLE_ACCESS_KEY="your-access-key" +#S3_COMPATIBLE_SECRET_KEY="your-secret-key" +#S3_COMPATIBLE_REGION="us-east-1" +#S3_COMPATIBLE_BUCKET="your-bucket-name" + +# Optional: Custom S3 endpoint (for non-AWS providers) +#S3_COMPATIBLE_ENDPOINT="https://s3.us-east-1.amazonaws.com" +# Examples: +# MinIO: "http://localhost:9000" or "https://minio.example.com" +# DigitalOcean: "https://nyc3.digitaloceanspaces.com" +# Backblaze B2: "https://s3.us-west-001.backblazeb2.com" +# Wasabi: "https://s3.us-east-1.wasabisys.com" + +# Optional: Public URL where files will be accessible (overrides auto-generated URL) +#S3_COMPATIBLE_PUBLIC_URL="https://cdn.example.com" +# Examples: +# AWS S3: "https://my-bucket.s3.us-east-1.amazonaws.com" +# MinIO: "https://cdn.example.com" (with reverse proxy) +# DigitalOcean: "https://my-space.nyc3.cdn.digitaloceanspaces.com" + +# Optional: Use path-style URLs (required for MinIO, optional for others) +#S3_COMPATIBLE_PATH_STYLE=false + +# Optional: Generate signed URLs for private buckets +#S3_COMPATIBLE_SIGNED_URLS=false +#S3_COMPATIBLE_SIGNED_URL_EXPIRY=3600 + +# === FTP STORAGE === +# Upload files via FTP protocol to any FTP server +#FTP_HOST="ftp.example.com" +#FTP_PORT=21 +#FTP_USER="ftpuser" +#FTP_PASSWORD="ftppassword" +#FTP_REMOTE_PATH="/public_html/uploads" +#FTP_PUBLIC_URL="https://example.com/uploads" + +# Optional: Use FTPS (FTP over SSL/TLS) +#FTP_SECURE=false + +# Optional: Use passive FTP mode (recommended for most setups) +#FTP_PASSIVE_MODE=true + +# === SFTP STORAGE === +# Upload files via SFTP (SSH File Transfer Protocol) +#SFTP_HOST="sftp.example.com" +#SFTP_PORT=22 +#SFTP_USER="sftpuser" +#SFTP_REMOTE_PATH="/var/www/uploads" +#SFTP_PUBLIC_URL="https://example.com/uploads" + +# Authentication: Use either password OR SSH key (not both) +#SFTP_PASSWORD="sftppassword" +#SFTP_PRIVATE_KEY_PATH="/path/to/private/key" # Social Media API Settings X_API_KEY="" diff --git a/.gitignore b/.gitignore index 9ad2f651a..ad00ef7da 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,9 @@ testem.log .DS_Store Thumbs.db +# macOS resource fork files +._* + .nx/cache .nx/workspace-data diff --git a/libraries/nestjs-libraries/src/upload/ftp.storage.ts b/libraries/nestjs-libraries/src/upload/ftp.storage.ts new file mode 100644 index 000000000..b2c05906c --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/ftp.storage.ts @@ -0,0 +1,243 @@ +import * as ftp from 'basic-ftp'; +import 'multer'; +import { makeId } from '@gitroom/nestjs-libraries/services/make.is'; +import mime from 'mime-types'; +// @ts-ignore +import { getExtension } from 'mime'; +import { IUploadProvider } from './upload.interface'; +import axios from 'axios'; +import { Readable } from 'stream'; + +/** + * FTP storage provider for uploading files via FTP protocol + * Supports both FTP and FTPS (FTP over SSL/TLS) + */ +export class FTPStorage implements IUploadProvider { + private _host: string; + private _port: number; + private _user: string; + private _password: string; + private _remotePath: string; + private _publicUrl: string; + private _secure: boolean; + private _passiveMode: boolean; + + /** + * Initialize FTP storage provider + * @param host - FTP server hostname + * @param port - FTP server port (default: 21) + * @param user - FTP username + * @param password - FTP password + * @param remotePath - Remote directory path on FTP server + * @param publicUrl - Public URL where uploaded files will be accessible via HTTP + * @param secure - Use FTPS (FTP over SSL/TLS) (default: false) + * @param passiveMode - Use passive FTP mode (default: true) + */ + constructor( + host: string, + port: number = 21, + user: string, + password: string, + remotePath: string, + publicUrl: string, + secure: boolean = false, + passiveMode: boolean = true + ) { + this._host = host; + this._port = port; + this._user = user; + this._password = password; + this._remotePath = remotePath.endsWith('/') ? remotePath.slice(0, -1) : remotePath; + this._publicUrl = publicUrl.endsWith('/') ? publicUrl.slice(0, -1) : publicUrl; + this._secure = secure; + this._passiveMode = passiveMode; + } + + /** + * Create and configure FTP client connection + * @returns Promise - Configured FTP client + */ + private async createFTPClient(): Promise { + const client = new ftp.Client(); + + try { + // Configure connection timeout and keep-alive + client.ftp.timeout = 30000; // 30 seconds timeout + + await client.access({ + host: this._host, + port: this._port, + user: this._user, + password: this._password, + secure: this._secure, + secureOptions: this._secure ? { rejectUnauthorized: false } : undefined, + }); + + // Set passive mode + client.ftp.pasv = this._passiveMode; + + return client; + } catch (error) { + client.close(); + throw new Error(`Failed to connect to FTP server: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Generate date-based directory structure (YYYY/MM/DD) + * @returns string - Directory path + */ + private generateDatePath(): string { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + return `${year}/${month}/${day}`; + } + + /** + * Ensure directory exists on FTP server, creating if necessary + * @param client - FTP client + * @param dirPath - Directory path to create + */ + private async ensureDirectory(client: ftp.Client, dirPath: string): Promise { + try { + await client.ensureDir(dirPath); + } catch (error) { + throw new Error(`Failed to create directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Upload a file from a URL via FTP + * @param path - URL of the file to upload + * @returns Promise - The public URL where the uploaded file can be accessed + */ + async uploadSimple(path: string): Promise { + let client: ftp.Client | null = null; + + try { + // Download file from URL + const response = await axios.get(path, { responseType: 'arraybuffer' }); + const contentType = response.headers['content-type'] || 'application/octet-stream'; + + const extension = getExtension(contentType); + if (!extension) { + throw new Error(`Unable to determine file extension for content type: ${contentType}`); + } + + // Generate file path + const datePath = this.generateDatePath(); + const id = makeId(10); + const fileName = `${id}.${extension}`; + const remoteDir = `${this._remotePath}/${datePath}`; + const remoteFilePath = `${remoteDir}/${fileName}`; + const publicPath = `${datePath}/${fileName}`; + + // Connect to FTP server + client = await this.createFTPClient(); + + // Ensure directory exists + await this.ensureDirectory(client, remoteDir); + + // Create readable stream from buffer + const stream = Readable.from(response.data); + + // Upload file + await client.uploadFrom(stream, remoteFilePath); + + return `${this._publicUrl}/${publicPath}`; + } catch (error) { + console.error('Error uploading file via FTP uploadSimple:', error); + throw new Error(`Failed to upload file from ${path}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + if (client) { + client.close(); + } + } + } + + /** + * Upload a file from form data via FTP + * @param file - The uploaded file object + * @returns Promise - File information including the public access URL + */ + async uploadFile(file: Express.Multer.File): Promise { + let client: ftp.Client | null = null; + + try { + // Generate file path + const datePath = this.generateDatePath(); + const id = makeId(10); + const extension = mime.extension(file.mimetype) || 'bin'; + const fileName = `${id}.${extension}`; + const remoteDir = `${this._remotePath}/${datePath}`; + const remoteFilePath = `${remoteDir}/${fileName}`; + const publicPath = `${datePath}/${fileName}`; + + // Connect to FTP server + client = await this.createFTPClient(); + + // Ensure directory exists + await this.ensureDirectory(client, remoteDir); + + // Create readable stream from buffer + const stream = Readable.from(file.buffer); + + // Upload file + await client.uploadFrom(stream, remoteFilePath); + + const publicUrl = `${this._publicUrl}/${publicPath}`; + + return { + filename: fileName, + mimetype: file.mimetype, + size: file.size, + buffer: file.buffer, + originalname: fileName, + fieldname: 'file', + path: publicUrl, + destination: publicUrl, + encoding: '7bit', + stream: file.buffer as any, + }; + } catch (error) { + console.error('Error uploading file via FTP uploadFile:', error); + throw new Error(`Failed to upload file ${file.originalname}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + if (client) { + client.close(); + } + } + } + + /** + * Remove a file from FTP storage + * @param filePath - The public URL of the file to remove + * @returns Promise + */ + async removeFile(filePath: string): Promise { + let client: ftp.Client | null = null; + + try { + // Extract relative path from public URL + const relativePath = filePath.replace(this._publicUrl + '/', ''); + const remoteFilePath = `${this._remotePath}/${relativePath}`; + + // Connect to FTP server + client = await this.createFTPClient(); + + // Remove file + await client.remove(remoteFilePath); + } catch (error) { + console.error('Error removing file via FTP:', error); + // Don't throw error for file removal failures to avoid breaking the application + // Just log the error as the file might already be deleted or not exist + console.warn(`Failed to remove file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + if (client) { + client.close(); + } + } + } +} \ No newline at end of file diff --git a/libraries/nestjs-libraries/src/upload/s3-compatible.storage.ts b/libraries/nestjs-libraries/src/upload/s3-compatible.storage.ts new file mode 100644 index 000000000..843b17958 --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/s3-compatible.storage.ts @@ -0,0 +1,229 @@ +import { S3Client, PutObjectCommand, DeleteObjectCommand, GetObjectCommand } from '@aws-sdk/client-s3'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; +import 'multer'; +import { makeId } from '@gitroom/nestjs-libraries/services/make.is'; +import mime from 'mime-types'; +// @ts-ignore +import { getExtension } from 'mime'; +import { IUploadProvider } from './upload.interface'; + +/** + * S3-compatible storage provider that works with any S3-compatible service + * Supports AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, Wasabi, and more + */ +export class S3CompatibleStorage implements IUploadProvider { + private _client: S3Client; + private _bucketName: string; + private _publicUrl: string; + private _pathStyle: boolean; + private _useSignedUrls: boolean; + private _signedUrlExpiry: number; + + /** + * Initialize S3-compatible storage provider + * @param accessKeyId - S3 access key ID + * @param secretAccessKey - S3 secret access key + * @param region - S3 region (e.g., 'us-east-1', 'auto' for Cloudflare) + * @param bucketName - S3 bucket name + * @param endpoint - Custom S3 endpoint (optional, for S3-compatible services) + * @param publicUrl - Public URL where files will be accessible + * @param pathStyle - Use path-style URLs (true for MinIO, false for AWS S3) + * @param useSignedUrls - Generate signed URLs for private buckets + * @param signedUrlExpiry - Signed URL expiry time in seconds (default: 3600) + */ + constructor( + accessKeyId: string, + secretAccessKey: string, + region: string, + bucketName: string, + endpoint?: string, + publicUrl?: string, + pathStyle: boolean = false, + useSignedUrls: boolean = false, + signedUrlExpiry: number = 3600 + ) { + this._bucketName = bucketName; + this._pathStyle = pathStyle; + this._useSignedUrls = useSignedUrls; + this._signedUrlExpiry = signedUrlExpiry; + + // Configure public URL for file access + if (publicUrl) { + this._publicUrl = publicUrl.endsWith('/') ? publicUrl.slice(0, -1) : publicUrl; + } else if (endpoint) { + // Auto-generate public URL based on endpoint and path style + this._publicUrl = pathStyle + ? `${endpoint}/${bucketName}` + : `${endpoint.replace('://', `://${bucketName}.`)}`; + } else { + // Default AWS S3 URL format + this._publicUrl = `https://${bucketName}.s3.${region}.amazonaws.com`; + } + + // Initialize S3 client with appropriate configuration + this._client = new S3Client({ + endpoint, + region, + credentials: { + accessKeyId, + secretAccessKey, + }, + forcePathStyle: pathStyle, // Required for MinIO and some S3-compatible services + requestChecksumCalculation: 'WHEN_REQUIRED', + }); + + // Add middleware for Cloudflare R2 compatibility (removes checksum headers) + if (endpoint?.includes('cloudflarestorage.com')) { + this._client.middlewareStack.add( + (next) => + async (args): Promise => { + const request = args.request as RequestInit; + const headers = request.headers as Record; + + // Remove checksum headers that cause issues with Cloudflare R2 + delete headers['x-amz-checksum-crc32']; + delete headers['x-amz-checksum-crc32c']; + delete headers['x-amz-checksum-sha1']; + delete headers['x-amz-checksum-sha256']; + + request.headers = headers; + return next(args); + }, + { step: 'build', name: 'customHeaders' } + ); + } + } + + /** + * Generate public URL for a file + * @param fileName - The file name/key in the bucket + * @returns Promise - The public URL or signed URL + */ + private async getFileUrl(fileName: string): Promise { + if (this._useSignedUrls) { + // Generate signed URL for private buckets + const command = new GetObjectCommand({ + Bucket: this._bucketName, + Key: fileName, + }); + + return await getSignedUrl(this._client, command, { + expiresIn: this._signedUrlExpiry, + }); + } else { + // Return public URL + return `${this._publicUrl}/${fileName}`; + } + } + + /** + * Upload a file from a URL + * @param path - URL of the file to upload + * @returns Promise - The URL where the uploaded file can be accessed + */ + async uploadSimple(path: string): Promise { + try { + // Fetch the file from the provided URL + const response = await fetch(path); + if (!response.ok) { + throw new Error(`Failed to fetch file from ${path}: ${response.statusText}`); + } + + const contentType = + response.headers.get('content-type') || + response.headers.get('Content-Type') || + 'application/octet-stream'; + + const extension = getExtension(contentType); + if (!extension) { + throw new Error(`Unable to determine file extension for content type: ${contentType}`); + } + + const id = makeId(10); + const fileName = `${id}.${extension}`; + + // Upload to S3-compatible storage + const command = new PutObjectCommand({ + Bucket: this._bucketName, + Key: fileName, + Body: Buffer.from(await response.arrayBuffer()), + ContentType: contentType, + ACL: this._useSignedUrls ? undefined : 'public-read', // Only set ACL for public access + }); + + await this._client.send(command); + + return await this.getFileUrl(fileName); + } catch (error) { + console.error('Error uploading file via uploadSimple:', error); + throw new Error(`Failed to upload file from ${path}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Upload a file from form data + * @param file - The uploaded file object + * @returns Promise - File information including the access URL + */ + async uploadFile(file: Express.Multer.File): Promise { + try { + const id = makeId(10); + const extension = mime.extension(file.mimetype) || 'bin'; + const fileName = `${id}.${extension}`; + + // Upload to S3-compatible storage + const command = new PutObjectCommand({ + Bucket: this._bucketName, + Key: fileName, + Body: file.buffer, + ContentType: file.mimetype, + ACL: this._useSignedUrls ? undefined : 'public-read', // Only set ACL for public access + }); + + await this._client.send(command); + + const fileUrl = await this.getFileUrl(fileName); + + return { + filename: fileName, + mimetype: file.mimetype, + size: file.size, + buffer: file.buffer, + originalname: fileName, + fieldname: 'file', + path: fileUrl, + destination: fileUrl, + encoding: '7bit', + stream: file.buffer as any, + }; + } catch (error) { + console.error('Error uploading file via uploadFile:', error); + throw new Error(`Failed to upload file ${file.originalname}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Remove a file from storage + * @param filePath - The file path/URL to remove + * @returns Promise + */ + async removeFile(filePath: string): Promise { + try { + // Extract filename from the full URL + const fileName = filePath.split('/').pop(); + if (!fileName) { + throw new Error(`Invalid file path: ${filePath}`); + } + + const command = new DeleteObjectCommand({ + Bucket: this._bucketName, + Key: fileName, + }); + + await this._client.send(command); + } catch (error) { + console.error('Error removing file:', error); + throw new Error(`Failed to remove file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } +} \ No newline at end of file diff --git a/libraries/nestjs-libraries/src/upload/sftp.storage.ts b/libraries/nestjs-libraries/src/upload/sftp.storage.ts new file mode 100644 index 000000000..2eba53df2 --- /dev/null +++ b/libraries/nestjs-libraries/src/upload/sftp.storage.ts @@ -0,0 +1,252 @@ +import SftpClient from 'ssh2-sftp-client'; +import 'multer'; +import { makeId } from '@gitroom/nestjs-libraries/services/make.is'; +import mime from 'mime-types'; +// @ts-ignore +import { getExtension } from 'mime'; +import { IUploadProvider } from './upload.interface'; +import axios from 'axios'; +import * as fs from 'fs'; + +/** + * SFTP storage provider for secure file transfer over SSH + * Supports password and SSH key authentication + */ +export class SFTPStorage implements IUploadProvider { + private _host: string; + private _port: number; + private _user: string; + private _password?: string; + private _privateKeyPath?: string; + private _remotePath: string; + private _publicUrl: string; + + /** + * Initialize SFTP storage provider + * @param host - SFTP server hostname + * @param port - SFTP server port (default: 22) + * @param user - SFTP username + * @param remotePath - Remote directory path on SFTP server + * @param publicUrl - Public URL where uploaded files will be accessible via HTTP + * @param password - SFTP password (optional if using SSH key) + * @param privateKeyPath - Path to SSH private key file (optional if using password) + */ + constructor( + host: string, + port: number = 22, + user: string, + remotePath: string, + publicUrl: string, + password?: string, + privateKeyPath?: string + ) { + this._host = host; + this._port = port; + this._user = user; + this._password = password; + this._privateKeyPath = privateKeyPath; + this._remotePath = remotePath.endsWith('/') ? remotePath.slice(0, -1) : remotePath; + this._publicUrl = publicUrl.endsWith('/') ? publicUrl.slice(0, -1) : publicUrl; + + // Validate authentication method + if (!password && !privateKeyPath) { + throw new Error('Either password or privateKeyPath must be provided for SFTP authentication'); + } + } + + /** + * Create and configure SFTP client connection + * @returns Promise - Configured SFTP client + */ + private async createSFTPClient(): Promise { + const client = new SftpClient(); + + try { + const connectConfig: any = { + host: this._host, + port: this._port, + username: this._user, + readyTimeout: 30000, // 30 seconds timeout + keepaliveInterval: 10000, // 10 seconds keepalive + }; + + // Configure authentication method + if (this._privateKeyPath) { + // SSH key authentication + try { + const privateKey = fs.readFileSync(this._privateKeyPath); + connectConfig.privateKey = privateKey; + } catch (error) { + throw new Error(`Failed to read private key from ${this._privateKeyPath}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } else if (this._password) { + // Password authentication + connectConfig.password = this._password; + } + + await client.connect(connectConfig); + return client; + } catch (error) { + await client.end(); + throw new Error(`Failed to connect to SFTP server: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + + /** + * Generate date-based directory structure (YYYY/MM/DD) + * @returns string - Directory path + */ + private generateDatePath(): string { + const now = new Date(); + const year = now.getFullYear(); + const month = String(now.getMonth() + 1).padStart(2, '0'); + const day = String(now.getDate()).padStart(2, '0'); + return `${year}/${month}/${day}`; + } + + /** + * Ensure directory exists on SFTP server, creating if necessary + * @param client - SFTP client + * @param dirPath - Directory path to create + */ + private async ensureDirectory(client: SftpClient, dirPath: string): Promise { + try { + // Try to create directory (will fail if it already exists, which is fine) + await client.mkdir(dirPath, true); // recursive = true + } catch (error) { + // Directory might already exist, check if it's accessible + try { + await client.list(dirPath); + } catch (listError) { + throw new Error(`Failed to create or access directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } + } + + /** + * Upload a file from a URL via SFTP + * @param path - URL of the file to upload + * @returns Promise - The public URL where the uploaded file can be accessed + */ + async uploadSimple(path: string): Promise { + let client: SftpClient | null = null; + + try { + // Download file from URL + const response = await axios.get(path, { responseType: 'arraybuffer' }); + const contentType = response.headers['content-type'] || 'application/octet-stream'; + + const extension = getExtension(contentType); + if (!extension) { + throw new Error(`Unable to determine file extension for content type: ${contentType}`); + } + + // Generate file path + const datePath = this.generateDatePath(); + const id = makeId(10); + const fileName = `${id}.${extension}`; + const remoteDir = `${this._remotePath}/${datePath}`; + const remoteFilePath = `${remoteDir}/${fileName}`; + const publicPath = `${datePath}/${fileName}`; + + // Connect to SFTP server + client = await this.createSFTPClient(); + + // Ensure directory exists + await this.ensureDirectory(client, remoteDir); + + // Upload file + await client.put(Buffer.from(response.data), remoteFilePath); + + return `${this._publicUrl}/${publicPath}`; + } catch (error) { + console.error('Error uploading file via SFTP uploadSimple:', error); + throw new Error(`Failed to upload file from ${path}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + if (client) { + await client.end(); + } + } + } + + /** + * Upload a file from form data via SFTP + * @param file - The uploaded file object + * @returns Promise - File information including the public access URL + */ + async uploadFile(file: Express.Multer.File): Promise { + let client: SftpClient | null = null; + + try { + // Generate file path + const datePath = this.generateDatePath(); + const id = makeId(10); + const extension = mime.extension(file.mimetype) || 'bin'; + const fileName = `${id}.${extension}`; + const remoteDir = `${this._remotePath}/${datePath}`; + const remoteFilePath = `${remoteDir}/${fileName}`; + const publicPath = `${datePath}/${fileName}`; + + // Connect to SFTP server + client = await this.createSFTPClient(); + + // Ensure directory exists + await this.ensureDirectory(client, remoteDir); + + // Upload file + await client.put(file.buffer, remoteFilePath); + + const publicUrl = `${this._publicUrl}/${publicPath}`; + + return { + filename: fileName, + mimetype: file.mimetype, + size: file.size, + buffer: file.buffer, + originalname: fileName, + fieldname: 'file', + path: publicUrl, + destination: publicUrl, + encoding: '7bit', + stream: file.buffer as any, + }; + } catch (error) { + console.error('Error uploading file via SFTP uploadFile:', error); + throw new Error(`Failed to upload file ${file.originalname}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + if (client) { + await client.end(); + } + } + } + + /** + * Remove a file from SFTP storage + * @param filePath - The public URL of the file to remove + * @returns Promise + */ + async removeFile(filePath: string): Promise { + let client: SftpClient | null = null; + + try { + // Extract relative path from public URL + const relativePath = filePath.replace(this._publicUrl + '/', ''); + const remoteFilePath = `${this._remotePath}/${relativePath}`; + + // Connect to SFTP server + client = await this.createSFTPClient(); + + // Remove file + await client.delete(remoteFilePath); + } catch (error) { + console.error('Error removing file via SFTP:', error); + // Don't throw error for file removal failures to avoid breaking the application + // Just log the error as the file might already be deleted or not exist + console.warn(`Failed to remove file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`); + } finally { + if (client) { + await client.end(); + } + } + } +} \ No newline at end of file diff --git a/libraries/nestjs-libraries/src/upload/upload.factory.ts b/libraries/nestjs-libraries/src/upload/upload.factory.ts index f89d310a8..08ea8cb20 100644 --- a/libraries/nestjs-libraries/src/upload/upload.factory.ts +++ b/libraries/nestjs-libraries/src/upload/upload.factory.ts @@ -1,15 +1,41 @@ import { CloudflareStorage } from './cloudflare.storage'; import { IUploadProvider } from './upload.interface'; import { LocalStorage } from './local.storage'; +import { S3CompatibleStorage } from './s3-compatible.storage'; +import { FTPStorage } from './ftp.storage'; +import { SFTPStorage } from './sftp.storage'; export class UploadFactory { + /** + * Create storage provider instance based on environment configuration + * @returns IUploadProvider - Configured storage provider + */ static createStorage(): IUploadProvider { const storageProvider = process.env.STORAGE_PROVIDER || 'local'; switch (storageProvider) { case 'local': - return new LocalStorage(process.env.UPLOAD_DIRECTORY!); + if (!process.env.UPLOAD_DIRECTORY) { + throw new Error('UPLOAD_DIRECTORY environment variable is required for local storage'); + } + return new LocalStorage(process.env.UPLOAD_DIRECTORY); + case 'cloudflare': + const requiredCloudflareVars = [ + 'CLOUDFLARE_ACCOUNT_ID', + 'CLOUDFLARE_ACCESS_KEY', + 'CLOUDFLARE_SECRET_ACCESS_KEY', + 'CLOUDFLARE_REGION', + 'CLOUDFLARE_BUCKETNAME', + 'CLOUDFLARE_BUCKET_URL' + ]; + + for (const varName of requiredCloudflareVars) { + if (!process.env[varName]) { + throw new Error(`${varName} environment variable is required for Cloudflare storage`); + } + } + return new CloudflareStorage( process.env.CLOUDFLARE_ACCOUNT_ID!, process.env.CLOUDFLARE_ACCESS_KEY!, @@ -18,8 +44,93 @@ export class UploadFactory { process.env.CLOUDFLARE_BUCKETNAME!, process.env.CLOUDFLARE_BUCKET_URL! ); + + case 's3-compatible': + const requiredS3Vars = [ + 'S3_COMPATIBLE_ACCESS_KEY', + 'S3_COMPATIBLE_SECRET_KEY', + 'S3_COMPATIBLE_REGION', + 'S3_COMPATIBLE_BUCKET' + ]; + + for (const varName of requiredS3Vars) { + if (!process.env[varName]) { + throw new Error(`${varName} environment variable is required for S3-compatible storage`); + } + } + + return new S3CompatibleStorage( + process.env.S3_COMPATIBLE_ACCESS_KEY!, + process.env.S3_COMPATIBLE_SECRET_KEY!, + process.env.S3_COMPATIBLE_REGION!, + process.env.S3_COMPATIBLE_BUCKET!, + process.env.S3_COMPATIBLE_ENDPOINT, // Optional custom endpoint + process.env.S3_COMPATIBLE_PUBLIC_URL, // Optional public URL + process.env.S3_COMPATIBLE_PATH_STYLE === 'true', // Path style (for MinIO) + process.env.S3_COMPATIBLE_SIGNED_URLS === 'true', // Use signed URLs + process.env.S3_COMPATIBLE_SIGNED_URL_EXPIRY ? parseInt(process.env.S3_COMPATIBLE_SIGNED_URL_EXPIRY) : 3600 + ); + + case 'ftp': + const requiredFtpVars = [ + 'FTP_HOST', + 'FTP_USER', + 'FTP_PASSWORD', + 'FTP_REMOTE_PATH', + 'FTP_PUBLIC_URL' + ]; + + for (const varName of requiredFtpVars) { + if (!process.env[varName]) { + throw new Error(`${varName} environment variable is required for FTP storage`); + } + } + + return new FTPStorage( + process.env.FTP_HOST!, + process.env.FTP_PORT ? parseInt(process.env.FTP_PORT) : 21, + process.env.FTP_USER!, + process.env.FTP_PASSWORD!, + process.env.FTP_REMOTE_PATH!, + process.env.FTP_PUBLIC_URL!, + process.env.FTP_SECURE === 'true', + process.env.FTP_PASSIVE_MODE !== 'false' // Default to true + ); + + case 'sftp': + const requiredSftpVars = [ + 'SFTP_HOST', + 'SFTP_USER', + 'SFTP_REMOTE_PATH', + 'SFTP_PUBLIC_URL' + ]; + + for (const varName of requiredSftpVars) { + if (!process.env[varName]) { + throw new Error(`${varName} environment variable is required for SFTP storage`); + } + } + + // Validate authentication method + if (!process.env.SFTP_PASSWORD && !process.env.SFTP_PRIVATE_KEY_PATH) { + throw new Error('Either SFTP_PASSWORD or SFTP_PRIVATE_KEY_PATH must be provided for SFTP authentication'); + } + + return new SFTPStorage( + process.env.SFTP_HOST!, + process.env.SFTP_PORT ? parseInt(process.env.SFTP_PORT) : 22, + process.env.SFTP_USER!, + process.env.SFTP_REMOTE_PATH!, + process.env.SFTP_PUBLIC_URL!, + process.env.SFTP_PASSWORD, + process.env.SFTP_PRIVATE_KEY_PATH + ); + default: - throw new Error(`Invalid storage type ${storageProvider}`); + throw new Error( + `Invalid storage provider '${storageProvider}'. ` + + `Supported providers: local, cloudflare, s3-compatible, ftp, sftp` + ); } } } diff --git a/package.json b/package.json index dd315d628..a9ff47d4a 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,8 @@ "@atproto/api": "^0.15.15", "@aws-sdk/client-s3": "^3.787.0", "@aws-sdk/s3-request-presigner": "^3.787.0", + "basic-ftp": "^5.0.5", + "ssh2-sftp-client": "^10.0.3", "@casl/ability": "^6.5.0", "@copilotkit/react-core": "^1.8.9", "@copilotkit/react-textarea": "^1.8.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7fd3804c4..30ae60be1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -264,6 +264,9 @@ importers: axios: specifier: ^1.7.7 version: 1.12.1(debug@4.4.3) + basic-ftp: + specifier: ^5.0.5 + version: 5.0.5 bcrypt: specifier: ^5.1.1 version: 5.1.1 @@ -495,6 +498,9 @@ importers: slugify: specifier: ^1.6.6 version: 1.6.6 + ssh2-sftp-client: + specifier: ^10.0.3 + version: 10.0.3 stripe: specifier: ^15.5.0 version: 15.12.0 @@ -7700,6 +7706,10 @@ packages: resolution: {integrity: sha512-WDtdLmJvAuNNPzByAYpRo2rF1Mmradw6gvWsQKf63476DDXmomT9zUiGypLcG4ibIM67vhAj8jJRdbmEws2Aqw==} engines: {node: '>=6.14.2'} + buildcheck@0.0.6: + resolution: {integrity: sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==} + engines: {node: '>=10.0.0'} + bullmq@5.58.5: resolution: {integrity: sha512-0A6Qjxdn8j7aOcxfRZY798vO/aMuwvoZwfE6a9EOXHb1pzpBVAogsc/OfRWeUf+5wMBoYB5nthstnJo/zrQOeQ==} @@ -8227,6 +8237,10 @@ packages: typescript: optional: true + cpu-features@0.0.10: + resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} + engines: {node: '>=10.0.0'} + crc@3.8.0: resolution: {integrity: sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==} @@ -8777,6 +8791,9 @@ packages: resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==} engines: {node: '>=0.12'} + err-code@2.0.3: + resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} + error-ex@1.3.2: resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} @@ -12603,6 +12620,10 @@ packages: resolution: {integrity: sha512-p/iXrPSVfnqPft24ZdNNLECw/UrtLTpT3jpAAMzl/o5/rDsGCPo3/CQS2611flL6LkoEJ3oQZw7C8Q80ZISXRQ==} engines: {node: '>= 0.8.0'} + promise-retry@2.0.1: + resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} + engines: {node: '>=10'} + promise@8.3.0: resolution: {integrity: sha512-rZPNPKTOYVNEEKFaq1HqTgOwZD+4/YHS5ukLzQCypkj+OkYx7iv0mA91lJlpPPZ8vMau3IIGj5Qlwrx+8iiSmg==} @@ -13813,6 +13834,14 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + ssh2-sftp-client@10.0.3: + resolution: {integrity: sha512-Wlhasz/OCgrlqC8IlBZhF19Uw/X/dHI8ug4sFQybPE+0sDztvgvDf7Om6o7LbRLe68E7XkFZf3qMnqAvqn1vkQ==} + engines: {node: '>=16.20.2'} + + ssh2@1.17.0: + resolution: {integrity: sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==} + engines: {node: '>=10.16.0'} + sshpk@1.18.0: resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} engines: {node: '>=0.10.0'} @@ -24910,6 +24939,9 @@ snapshots: dependencies: node-gyp-build: 4.8.4 + buildcheck@0.0.6: + optional: true + bullmq@5.58.5: dependencies: cron-parser: 4.9.0 @@ -25473,6 +25505,12 @@ snapshots: optionalDependencies: typescript: 5.5.4 + cpu-features@0.0.10: + dependencies: + buildcheck: 0.0.6 + nan: 2.23.0 + optional: true + crc@3.8.0: dependencies: buffer: 5.7.1 @@ -26044,6 +26082,8 @@ snapshots: entities@6.0.1: {} + err-code@2.0.3: {} + error-ex@1.3.2: dependencies: is-arrayish: 0.2.1 @@ -31217,6 +31257,11 @@ snapshots: promise-queue@2.2.5: {} + promise-retry@2.0.1: + dependencies: + err-code: 2.0.3 + retry: 0.12.0 + promise@8.3.0: dependencies: asap: 2.0.6 @@ -32835,6 +32880,20 @@ snapshots: sprintf-js@1.0.3: {} + ssh2-sftp-client@10.0.3: + dependencies: + concat-stream: 2.0.0 + promise-retry: 2.0.1 + ssh2: 1.17.0 + + ssh2@1.17.0: + dependencies: + asn1: 0.2.6 + bcrypt-pbkdf: 1.0.2 + optionalDependencies: + cpu-features: 0.0.10 + nan: 2.23.0 + sshpk@1.18.0: dependencies: asn1: 0.2.6 diff --git a/scripts/sync-upstream.sh b/scripts/sync-upstream.sh new file mode 100644 index 000000000..09a051ed3 --- /dev/null +++ b/scripts/sync-upstream.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +# Postiz App - Upstream Sync Script +# This script helps manage syncing between upstream, public fork, and private repository + +set -e + +echo "🔄 Starting upstream sync process..." + +# Colors for output +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +# Function to print colored output +print_status() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Check if we're in a git repository +if ! git rev-parse --git-dir > /dev/null 2>&1; then + print_error "Not in a git repository!" + exit 1 +fi + +# Check if all remotes exist +check_remotes() { + print_status "Checking remotes..." + + if ! git remote | grep -q "^upstream$"; then + print_error "upstream remote not found. Please add it with:" + echo "git remote add upstream https://github.com/gitroomhq/postiz-app.git" + exit 1 + fi + + if ! git remote | grep -q "^origin$"; then + print_error "origin remote not found." + exit 1 + fi + + if ! git remote | grep -q "^private$"; then + print_error "private remote not found. Please add it with:" + echo "git remote add private https://github.com/gabelul/postiz-app-private.git" + exit 1 + fi + + print_success "All remotes configured correctly" +} + +# Fetch all remotes +fetch_all() { + print_status "Fetching from all remotes..." + git fetch upstream + git fetch origin + git fetch private + print_success "Fetched all remotes" +} + +# Sync public fork with upstream +sync_public_fork() { + print_status "Syncing public fork (main) with upstream..." + + # Switch to main branch + git checkout main + + # Check if there are uncommitted changes + if ! git diff --quiet || ! git diff --cached --quiet; then + print_warning "Uncommitted changes detected. Please commit or stash them first." + return 1 + fi + + # Merge upstream changes + git merge upstream/main + + # Push to public fork + git push origin main + + print_success "Public fork synced with upstream" +} + +# Show differences between upstream and private +show_diff() { + print_status "Showing differences between upstream/main and private-main..." + + git checkout private-main + echo -e "${BLUE}Files modified in private version:${NC}" + git diff --name-only upstream/main..private-main + + echo -e "${BLUE}Commit differences:${NC}" + git log --oneline upstream/main..private-main +} + +# Merge upstream changes to private branch +merge_to_private() { + print_status "Merging upstream changes to private-main..." + + git checkout private-main + + # Check if there are uncommitted changes + if ! git diff --quiet || ! git diff --cached --quiet; then + print_warning "Uncommitted changes detected. Please commit or stash them first." + return 1 + fi + + # Create a backup branch before merging + backup_branch="private-main-backup-$(date +%Y%m%d-%H%M%S)" + git branch $backup_branch + print_status "Created backup branch: $backup_branch" + + # Merge upstream changes + if git merge upstream/main; then + print_success "Successfully merged upstream changes to private-main" + + # Push to private repository + git push private private-main + print_success "Pushed updated private-main to private repository" + + # Clean up backup branch + git branch -d $backup_branch + print_status "Cleaned up backup branch" + else + print_error "Merge conflicts detected. Please resolve them manually." + print_status "Backup branch created: $backup_branch" + echo "After resolving conflicts, run: git push private private-main" + return 1 + fi +} + +# Main function to handle command line arguments +main() { + case "${1:-}" in + "check") + check_remotes + ;; + "fetch") + check_remotes + fetch_all + ;; + "sync-public") + check_remotes + fetch_all + sync_public_fork + ;; + "show-diff") + check_remotes + fetch_all + show_diff + ;; + "merge-private") + check_remotes + fetch_all + merge_to_private + ;; + "full-sync") + check_remotes + fetch_all + sync_public_fork + show_diff + echo -e "${YELLOW}Review the differences above. Run with 'merge-private' to merge upstream changes to private branch.${NC}" + ;; + *) + echo "Usage: $0 {check|fetch|sync-public|show-diff|merge-private|full-sync}" + echo "" + echo "Commands:" + echo " check - Verify all remotes are configured" + echo " fetch - Fetch from all remotes" + echo " sync-public - Sync public fork (main) with upstream" + echo " show-diff - Show differences between upstream and private" + echo " merge-private - Merge upstream changes to private-main" + echo " full-sync - Run sync-public and show-diff" + echo "" + echo "Typical workflow:" + echo " 1. $0 full-sync" + echo " 2. Review differences" + echo " 3. $0 merge-private (if you want to merge upstream changes)" + exit 1 + ;; + esac +} + +main "$@" \ No newline at end of file