From cc914cc4be1a53a1bb0640a468ab64bb66120720 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Tue, 1 Jul 2025 12:24:23 +0200 Subject: [PATCH 1/5] dev --- packages/cubejs-prestodb-driver/src/PrestoDriver.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts index fc600e5dd45e8..5c5ab2fc61f23 100644 --- a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts +++ b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts @@ -30,7 +30,7 @@ const presto = require('presto-client'); export type PrestoDriverExportBucket = { exportBucket?: string, - bucketType?: 'gcs', + bucketType?: 'gcs' | 's3', credentials?: any, exportBucketCsvEscapeSymbol?: string, }; @@ -50,7 +50,7 @@ export type PrestoDriverConfiguration = PrestoDriverExportBucket & { queryTimeout?: number; }; -const SUPPORTED_BUCKET_TYPES = ['gcs']; +const SUPPORTED_BUCKET_TYPES = ['gcs', 's3']; /** * Presto driver class. */ @@ -103,7 +103,7 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { ...(authToken ? { custom_auth: `Bearer ${authToken}` } : {}), ...(dbPassword ? { basic_auth: { user: dbUser, password: dbPassword } } : {}), ssl: this.getSslOptions(dataSource), - bucketType: getEnv('dbExportBucketType', { supported: ['gcs'], dataSource }), + bucketType: getEnv('dbExportBucketType', { supported: SUPPORTED_BUCKET_TYPES, dataSource }), exportBucket: getEnv('dbExportBucket', { dataSource }), credentials: getEnv('dbExportGCSCredentials', { dataSource }), ...config @@ -351,6 +351,8 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { switch (bucketType) { case 'gcs': return this.extractFilesFromGCS({ credentials }, exportBucket, `${schema}/${tableName}`); + case 's3': + return this.extractUnloadedFilesFromS3({ credentials }, exportBucket, `${schema}/${tableName}`); default: throw new Error(`Unsupported export bucket type: ${bucketType}`); } From acfc4a989166f2b22ba9b91d755115656fde4f77 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Tue, 1 Jul 2025 14:59:14 +0200 Subject: [PATCH 2/5] dev --- .../src/PrestoDriver.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts index 5c5ab2fc61f23..c342ed385d5b2 100644 --- a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts +++ b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts @@ -32,6 +32,9 @@ export type PrestoDriverExportBucket = { exportBucket?: string, bucketType?: 'gcs' | 's3', credentials?: any, + accessKeyId?: string, + secretAccessKey?: string, + exportBucketRegion?: string, exportBucketCsvEscapeSymbol?: string, }; @@ -105,6 +108,9 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { ssl: this.getSslOptions(dataSource), bucketType: getEnv('dbExportBucketType', { supported: SUPPORTED_BUCKET_TYPES, dataSource }), exportBucket: getEnv('dbExportBucket', { dataSource }), + accessKeyId: getEnv('dbExportBucketAwsKey', { dataSource }), + secretAccessKey: getEnv('dbExportBucketAwsSecret', { dataSource }), + exportBucketRegion: getEnv('dbExportBucketAwsRegion', { dataSource }), credentials: getEnv('dbExportGCSCredentials', { dataSource }), ...config }; @@ -312,7 +318,11 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { const { schema, tableName } = this.splitTableFullName(params.tableFullName); const tableWithCatalogAndSchema = `${this.config.catalog}.${schema}.${tableName}`; - const protocol = bucketType === 'gcs' ? 'gs' : bucketType; + let protocol = bucketType === 'gcs' ? 'gs' : bucketType; + if (bucketType === 's3') { + protocol = 's3a'; + } + const externalLocation = `${protocol}://${exportBucket}/${schema}/${tableName}`; const withParams = `( external_location = '${externalLocation}', format = 'CSV')`; const select = `SELECT ${this.generateTableColumnsForExport(types)} FROM (${params.fromSql})`; @@ -345,14 +355,21 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { if (!this.config.exportBucket) { throw new Error('Export bucket is not configured.'); } - const { bucketType, exportBucket, credentials } = this.config; + const { bucketType, exportBucket } = this.config; const { schema, tableName } = this.splitTableFullName(tableFullName); switch (bucketType) { case 'gcs': - return this.extractFilesFromGCS({ credentials }, exportBucket, `${schema}/${tableName}`); + return this.extractFilesFromGCS({ credentials: this.config.credentials }, exportBucket, `${schema}/${tableName}`); case 's3': - return this.extractUnloadedFilesFromS3({ credentials }, exportBucket, `${schema}/${tableName}`); + return this.extractUnloadedFilesFromS3({ + credentials: { + accessKeyId: this.config.accessKeyId || '', + secretAccessKey: this.config.secretAccessKey || '', + }, + region: this.config.exportBucketRegion, + }, + exportBucket, `${schema}/${tableName}`); default: throw new Error(`Unsupported export bucket type: ${bucketType}`); } From 04224926fae1a5f8564574a4b618db7698c1c5f1 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Tue, 1 Jul 2025 15:05:00 +0200 Subject: [PATCH 3/5] dev --- packages/cubejs-prestodb-driver/src/PrestoDriver.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts index c342ed385d5b2..e8564395d23ca 100644 --- a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts +++ b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts @@ -35,6 +35,7 @@ export type PrestoDriverExportBucket = { accessKeyId?: string, secretAccessKey?: string, exportBucketRegion?: string, + exportBucketS3AdvancedFS?: boolean, exportBucketCsvEscapeSymbol?: string, }; @@ -318,10 +319,11 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { const { schema, tableName } = this.splitTableFullName(params.tableFullName); const tableWithCatalogAndSchema = `${this.config.catalog}.${schema}.${tableName}`; - let protocol = bucketType === 'gcs' ? 'gs' : bucketType; - if (bucketType === 's3') { - protocol = 's3a'; - } + + const protocol = { + gcs: 'gc', + s3: this.config.exportBucketS3AdvancedFS ? 's3a' : 's3' + }[bucketType || 'gcs']; const externalLocation = `${protocol}://${exportBucket}/${schema}/${tableName}`; const withParams = `( external_location = '${externalLocation}', format = 'CSV')`; From 0a834269f0f7b4c1d00f1befe41fc1085b582a52 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Tue, 1 Jul 2025 15:07:32 +0200 Subject: [PATCH 4/5] fix --- packages/cubejs-prestodb-driver/src/PrestoDriver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts index e8564395d23ca..313d77227dfb4 100644 --- a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts +++ b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts @@ -321,7 +321,7 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { const tableWithCatalogAndSchema = `${this.config.catalog}.${schema}.${tableName}`; const protocol = { - gcs: 'gc', + gcs: 'gs', s3: this.config.exportBucketS3AdvancedFS ? 's3a' : 's3' }[bucketType || 'gcs']; From 48d112b8152642e27a0b901bc9a9c873d3fa7b23 Mon Sep 17 00:00:00 2001 From: Dmitriy Rusov Date: Tue, 1 Jul 2025 15:11:30 +0200 Subject: [PATCH 5/5] dev --- packages/cubejs-prestodb-driver/src/PrestoDriver.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts index 313d77227dfb4..2490d99473fdb 100644 --- a/packages/cubejs-prestodb-driver/src/PrestoDriver.ts +++ b/packages/cubejs-prestodb-driver/src/PrestoDriver.ts @@ -365,10 +365,12 @@ export class PrestoDriver extends BaseDriver implements DriverInterface { return this.extractFilesFromGCS({ credentials: this.config.credentials }, exportBucket, `${schema}/${tableName}`); case 's3': return this.extractUnloadedFilesFromS3({ - credentials: { - accessKeyId: this.config.accessKeyId || '', - secretAccessKey: this.config.secretAccessKey || '', - }, + credentials: this.config.accessKeyId && this.config.secretAccessKey + ? { + accessKeyId: this.config.accessKeyId, + secretAccessKey: this.config.secretAccessKey, + } + : undefined, region: this.config.exportBucketRegion, }, exportBucket, `${schema}/${tableName}`);