Skip to content

Commit 705f4b4

Browse files
author
Gabi
committed
feat: add comprehensive storage provider system
- Add generic S3-compatible storage provider supporting any S3-compatible service - Supports AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, Wasabi, and more - Configurable public URLs with support for CDNs and custom domains - Signed URL generation for private buckets - Path-style and virtual-hosted-style URL support - Cloudflare R2 compatibility with checksum header handling - Add FTP storage provider for traditional file transfer - Supports both FTP and FTPS (FTP over SSL/TLS) - Configurable passive/active modes - Connection pooling and timeout handling - Date-based directory organization - Add SFTP storage provider for secure file transfer - SSH key and password authentication support - Connection keepalive and timeout management - Secure file transfer over SSH - Update upload factory with comprehensive error handling - Detailed environment variable validation - Clear error messages for missing configuration - Support for all storage providers: local, cloudflare, s3-compatible, ftp, sftp - Comprehensive environment configuration documentation - Detailed .env.example with examples for all providers - Clear separation between upload credentials and public URLs - Provider-specific configuration options Closes gitroomhq#322 Supersedes gitroomhq#873
1 parent 84601a6 commit 705f4b4

File tree

7 files changed

+972
-8
lines changed

7 files changed

+972
-8
lines changed

.env.example

Lines changed: 74 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,82 @@ CLOUDFLARE_REGION="auto"
3030
#EMAIL_FROM_NAME=""
3131
#DISABLE_REGISTRATION=false
3232

33-
# Where will social media icons be saved - local or cloudflare.
33+
# ===================================================================
34+
# STORAGE CONFIGURATION
35+
# ===================================================================
36+
# Where will social media icons and uploaded files be saved.
37+
# Supported providers: local, cloudflare, s3-compatible, ftp, sftp
3438
STORAGE_PROVIDER="local"
3539

36-
# Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
37-
#UPLOAD_DIRECTORY=""
38-
39-
# Your upload directory path if you host your files locally, otherwise Cloudflare will be used.
40-
#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY=""
40+
# === LOCAL STORAGE ===
41+
# Your upload directory path if you host your files locally
42+
#UPLOAD_DIRECTORY="/path/to/upload/directory"
43+
#NEXT_PUBLIC_UPLOAD_STATIC_DIRECTORY="/uploads"
44+
45+
# === CLOUDFLARE R2 STORAGE ===
46+
# Cloudflare R2 is S3-compatible and used for the existing cloudflare provider
47+
#CLOUDFLARE_ACCOUNT_ID="your-account-id"
48+
#CLOUDFLARE_ACCESS_KEY="your-access-key"
49+
#CLOUDFLARE_SECRET_ACCESS_KEY="your-secret-access-key"
50+
#CLOUDFLARE_BUCKETNAME="your-bucket-name"
51+
#CLOUDFLARE_BUCKET_URL="https://your-bucket-url.r2.cloudflarestorage.com/"
52+
#CLOUDFLARE_REGION="auto"
53+
54+
# === S3-COMPATIBLE STORAGE ===
55+
# Works with AWS S3, MinIO, DigitalOcean Spaces, Backblaze B2, Wasabi, and more
56+
#S3_COMPATIBLE_ACCESS_KEY="your-access-key"
57+
#S3_COMPATIBLE_SECRET_KEY="your-secret-key"
58+
#S3_COMPATIBLE_REGION="us-east-1"
59+
#S3_COMPATIBLE_BUCKET="your-bucket-name"
60+
61+
# Optional: Custom S3 endpoint (for non-AWS providers)
62+
#S3_COMPATIBLE_ENDPOINT="https://s3.us-east-1.amazonaws.com"
63+
# Examples:
64+
# MinIO: "http://localhost:9000" or "https://minio.example.com"
65+
# DigitalOcean: "https://nyc3.digitaloceanspaces.com"
66+
# Backblaze B2: "https://s3.us-west-001.backblazeb2.com"
67+
# Wasabi: "https://s3.us-east-1.wasabisys.com"
68+
69+
# Optional: Public URL where files will be accessible (overrides auto-generated URL)
70+
#S3_COMPATIBLE_PUBLIC_URL="https://cdn.example.com"
71+
# Examples:
72+
# AWS S3: "https://my-bucket.s3.us-east-1.amazonaws.com"
73+
# MinIO: "https://cdn.example.com" (with reverse proxy)
74+
# DigitalOcean: "https://my-space.nyc3.cdn.digitaloceanspaces.com"
75+
76+
# Optional: Use path-style URLs (required for MinIO, optional for others)
77+
#S3_COMPATIBLE_PATH_STYLE=false
78+
79+
# Optional: Generate signed URLs for private buckets
80+
#S3_COMPATIBLE_SIGNED_URLS=false
81+
#S3_COMPATIBLE_SIGNED_URL_EXPIRY=3600
82+
83+
# === FTP STORAGE ===
84+
# Upload files via FTP protocol to any FTP server
85+
#FTP_HOST="ftp.example.com"
86+
#FTP_PORT=21
87+
#FTP_USER="ftpuser"
88+
#FTP_PASSWORD="ftppassword"
89+
#FTP_REMOTE_PATH="/public_html/uploads"
90+
#FTP_PUBLIC_URL="https://example.com/uploads"
91+
92+
# Optional: Use FTPS (FTP over SSL/TLS)
93+
#FTP_SECURE=false
94+
95+
# Optional: Use passive FTP mode (recommended for most setups)
96+
#FTP_PASSIVE_MODE=true
97+
98+
# === SFTP STORAGE ===
99+
# Upload files via SFTP (SSH File Transfer Protocol)
100+
#SFTP_HOST="sftp.example.com"
101+
#SFTP_PORT=22
102+
#SFTP_USER="sftpuser"
103+
#SFTP_REMOTE_PATH="/var/www/uploads"
104+
#SFTP_PUBLIC_URL="https://example.com/uploads"
105+
106+
# Authentication: Use either password OR SSH key (not both)
107+
#SFTP_PASSWORD="sftppassword"
108+
#SFTP_PRIVATE_KEY_PATH="/path/to/private/key"
41109

42110
# Social Media API Settings
43111
X_API_KEY=""
Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
import * as ftp from 'basic-ftp';
2+
import 'multer';
3+
import { makeId } from '@gitroom/nestjs-libraries/services/make.is';
4+
import mime from 'mime-types';
5+
// @ts-ignore
6+
import { getExtension } from 'mime';
7+
import { IUploadProvider } from './upload.interface';
8+
import axios from 'axios';
9+
import { Readable } from 'stream';
10+
11+
/**
12+
* FTP storage provider for uploading files via FTP protocol
13+
* Supports both FTP and FTPS (FTP over SSL/TLS)
14+
*/
15+
export class FTPStorage implements IUploadProvider {
16+
private _host: string;
17+
private _port: number;
18+
private _user: string;
19+
private _password: string;
20+
private _remotePath: string;
21+
private _publicUrl: string;
22+
private _secure: boolean;
23+
private _passiveMode: boolean;
24+
25+
/**
26+
* Initialize FTP storage provider
27+
* @param host - FTP server hostname
28+
* @param port - FTP server port (default: 21)
29+
* @param user - FTP username
30+
* @param password - FTP password
31+
* @param remotePath - Remote directory path on FTP server
32+
* @param publicUrl - Public URL where uploaded files will be accessible via HTTP
33+
* @param secure - Use FTPS (FTP over SSL/TLS) (default: false)
34+
* @param passiveMode - Use passive FTP mode (default: true)
35+
*/
36+
constructor(
37+
host: string,
38+
port: number = 21,
39+
user: string,
40+
password: string,
41+
remotePath: string,
42+
publicUrl: string,
43+
secure: boolean = false,
44+
passiveMode: boolean = true
45+
) {
46+
this._host = host;
47+
this._port = port;
48+
this._user = user;
49+
this._password = password;
50+
this._remotePath = remotePath.endsWith('/') ? remotePath.slice(0, -1) : remotePath;
51+
this._publicUrl = publicUrl.endsWith('/') ? publicUrl.slice(0, -1) : publicUrl;
52+
this._secure = secure;
53+
this._passiveMode = passiveMode;
54+
}
55+
56+
/**
57+
* Create and configure FTP client connection
58+
* @returns Promise<ftp.Client> - Configured FTP client
59+
*/
60+
private async createFTPClient(): Promise<ftp.Client> {
61+
const client = new ftp.Client();
62+
63+
try {
64+
// Configure connection timeout and keep-alive
65+
client.ftp.timeout = 30000; // 30 seconds timeout
66+
67+
await client.access({
68+
host: this._host,
69+
port: this._port,
70+
user: this._user,
71+
password: this._password,
72+
secure: this._secure,
73+
secureOptions: this._secure ? { rejectUnauthorized: false } : undefined,
74+
});
75+
76+
// Set passive mode
77+
client.ftp.pasv = this._passiveMode;
78+
79+
return client;
80+
} catch (error) {
81+
client.close();
82+
throw new Error(`Failed to connect to FTP server: ${error instanceof Error ? error.message : 'Unknown error'}`);
83+
}
84+
}
85+
86+
/**
87+
* Generate date-based directory structure (YYYY/MM/DD)
88+
* @returns string - Directory path
89+
*/
90+
private generateDatePath(): string {
91+
const now = new Date();
92+
const year = now.getFullYear();
93+
const month = String(now.getMonth() + 1).padStart(2, '0');
94+
const day = String(now.getDate()).padStart(2, '0');
95+
return `${year}/${month}/${day}`;
96+
}
97+
98+
/**
99+
* Ensure directory exists on FTP server, creating if necessary
100+
* @param client - FTP client
101+
* @param dirPath - Directory path to create
102+
*/
103+
private async ensureDirectory(client: ftp.Client, dirPath: string): Promise<void> {
104+
try {
105+
await client.ensureDir(dirPath);
106+
} catch (error) {
107+
throw new Error(`Failed to create directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
108+
}
109+
}
110+
111+
/**
112+
* Upload a file from a URL via FTP
113+
* @param path - URL of the file to upload
114+
* @returns Promise<string> - The public URL where the uploaded file can be accessed
115+
*/
116+
async uploadSimple(path: string): Promise<string> {
117+
let client: ftp.Client | null = null;
118+
119+
try {
120+
// Download file from URL
121+
const response = await axios.get(path, { responseType: 'arraybuffer' });
122+
const contentType = response.headers['content-type'] || 'application/octet-stream';
123+
124+
const extension = getExtension(contentType);
125+
if (!extension) {
126+
throw new Error(`Unable to determine file extension for content type: ${contentType}`);
127+
}
128+
129+
// Generate file path
130+
const datePath = this.generateDatePath();
131+
const id = makeId(10);
132+
const fileName = `${id}.${extension}`;
133+
const remoteDir = `${this._remotePath}/${datePath}`;
134+
const remoteFilePath = `${remoteDir}/${fileName}`;
135+
const publicPath = `${datePath}/${fileName}`;
136+
137+
// Connect to FTP server
138+
client = await this.createFTPClient();
139+
140+
// Ensure directory exists
141+
await this.ensureDirectory(client, remoteDir);
142+
143+
// Create readable stream from buffer
144+
const stream = Readable.from(response.data);
145+
146+
// Upload file
147+
await client.uploadFrom(stream, remoteFilePath);
148+
149+
return `${this._publicUrl}/${publicPath}`;
150+
} catch (error) {
151+
console.error('Error uploading file via FTP uploadSimple:', error);
152+
throw new Error(`Failed to upload file from ${path}: ${error instanceof Error ? error.message : 'Unknown error'}`);
153+
} finally {
154+
if (client) {
155+
client.close();
156+
}
157+
}
158+
}
159+
160+
/**
161+
* Upload a file from form data via FTP
162+
* @param file - The uploaded file object
163+
* @returns Promise<any> - File information including the public access URL
164+
*/
165+
async uploadFile(file: Express.Multer.File): Promise<any> {
166+
let client: ftp.Client | null = null;
167+
168+
try {
169+
// Generate file path
170+
const datePath = this.generateDatePath();
171+
const id = makeId(10);
172+
const extension = mime.extension(file.mimetype) || 'bin';
173+
const fileName = `${id}.${extension}`;
174+
const remoteDir = `${this._remotePath}/${datePath}`;
175+
const remoteFilePath = `${remoteDir}/${fileName}`;
176+
const publicPath = `${datePath}/${fileName}`;
177+
178+
// Connect to FTP server
179+
client = await this.createFTPClient();
180+
181+
// Ensure directory exists
182+
await this.ensureDirectory(client, remoteDir);
183+
184+
// Create readable stream from buffer
185+
const stream = Readable.from(file.buffer);
186+
187+
// Upload file
188+
await client.uploadFrom(stream, remoteFilePath);
189+
190+
const publicUrl = `${this._publicUrl}/${publicPath}`;
191+
192+
return {
193+
filename: fileName,
194+
mimetype: file.mimetype,
195+
size: file.size,
196+
buffer: file.buffer,
197+
originalname: fileName,
198+
fieldname: 'file',
199+
path: publicUrl,
200+
destination: publicUrl,
201+
encoding: '7bit',
202+
stream: file.buffer as any,
203+
};
204+
} catch (error) {
205+
console.error('Error uploading file via FTP uploadFile:', error);
206+
throw new Error(`Failed to upload file ${file.originalname}: ${error instanceof Error ? error.message : 'Unknown error'}`);
207+
} finally {
208+
if (client) {
209+
client.close();
210+
}
211+
}
212+
}
213+
214+
/**
215+
* Remove a file from FTP storage
216+
* @param filePath - The public URL of the file to remove
217+
* @returns Promise<void>
218+
*/
219+
async removeFile(filePath: string): Promise<void> {
220+
let client: ftp.Client | null = null;
221+
222+
try {
223+
// Extract relative path from public URL
224+
const relativePath = filePath.replace(this._publicUrl + '/', '');
225+
const remoteFilePath = `${this._remotePath}/${relativePath}`;
226+
227+
// Connect to FTP server
228+
client = await this.createFTPClient();
229+
230+
// Remove file
231+
await client.remove(remoteFilePath);
232+
} catch (error) {
233+
console.error('Error removing file via FTP:', error);
234+
// Don't throw error for file removal failures to avoid breaking the application
235+
// Just log the error as the file might already be deleted or not exist
236+
console.warn(`Failed to remove file ${filePath}: ${error instanceof Error ? error.message : 'Unknown error'}`);
237+
} finally {
238+
if (client) {
239+
client.close();
240+
}
241+
}
242+
}
243+
}

0 commit comments

Comments
 (0)