From 2dd79a5b425d8806f0452e329e78c9eb034069fc Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 11:08:12 -0400 Subject: [PATCH 01/16] make keyId and secretKey optional for s3 --- .../src/SnowflakeDriver.ts | 34 +++++++++++++++---- .../fixtures/snowflake.json | 9 +++++ ...ke-export-bucket-s3-iam-roles-full.test.ts | 9 +++++ 3 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts index 3f8dcc6f329d2..2de7cce055ecf 100644 --- a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts +++ b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts @@ -106,8 +106,8 @@ const SnowflakeToGenericType: Record = { interface SnowflakeDriverExportAWS { bucketType: 's3', bucketName: string, - keyId: string, - secretKey: string, + keyId?: string, + secretKey?: string, region: string, integrationName?: string, } @@ -328,14 +328,17 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { if (bucketType === 's3') { // integrationName is optional for s3 const integrationName = getEnv('dbExportIntegration', { dataSource }); + // keyId and secretKey are optional for s3 if IAM role is used + const keyId = getEnv('dbExportBucketAwsKey', { dataSource }); + const secretKey = getEnv('dbExportBucketAwsSecret', { dataSource }); return { bucketType, bucketName: getEnv('dbExportBucket', { dataSource }), - keyId: getEnv('dbExportBucketAwsKey', { dataSource }), - secretKey: getEnv('dbExportBucketAwsSecret', { dataSource }), region: getEnv('dbExportBucketAwsRegion', { dataSource }), ...(integrationName !== undefined && { integrationName }), + ...(keyId !== undefined && { keyId }), + ...(secretKey !== undefined && { secretKey }), }; } @@ -387,6 +390,23 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { ); } + private getRequiredExportBucketKeys( + exportBucket: SnowflakeDriverExportBucket, + emptyKeys: string[] + ): string[] { + if (exportBucket.bucketType === 's3') { + const s3Config = exportBucket as SnowflakeDriverExportAWS; + const hasCredentials = s3Config.keyId && s3Config.secretKey; + const hasIntegration = s3Config.integrationName; + + if (!hasCredentials && !hasIntegration) { + return emptyKeys.filter(key => key !== 'keyId' && key !== 'secretKey'); + } + } + + return emptyKeys; + } + protected getExportBucket( dataSource: string, ): SnowflakeDriverExportBucket | undefined { @@ -402,9 +422,11 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { const emptyKeys = Object.keys(exportBucket) .filter((key: string) => exportBucket[key] === undefined); - if (emptyKeys.length) { + const keysToValidate = this.getRequiredExportBucketKeys(exportBucket, emptyKeys); + + if (keysToValidate.length) { throw new Error( - `Unsupported configuration exportBucket, some configuration keys are empty: ${emptyKeys.join(',')}` + `Unsupported configuration exportBucket, some configuration keys are empty: ${keysToValidate.join(',')}` ); } diff --git a/packages/cubejs-testing-drivers/fixtures/snowflake.json b/packages/cubejs-testing-drivers/fixtures/snowflake.json index 29d46e844f592..83a6daa5080d6 100644 --- a/packages/cubejs-testing-drivers/fixtures/snowflake.json +++ b/packages/cubejs-testing-drivers/fixtures/snowflake.json @@ -22,6 +22,15 @@ } } }, + "export-bucket-s3-iam-roles": { + "cube": { + "environment": { + "CUBEJS_DB_EXPORT_BUCKET_TYPE": "s3", + "CUBEJS_DB_EXPORT_BUCKET": "snowflake-drivers-tests-preaggs", + "CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-west-1" + } + } + }, "export-bucket-azure": { "cube": { "environment": { diff --git a/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts b/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts new file mode 100644 index 0000000000000..0a6649283d62e --- /dev/null +++ b/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts @@ -0,0 +1,9 @@ +import { testQueries } from '../src/tests/testQueries'; + +testQueries('snowflake', { + // NOTICE: It's enough to turn on this flag only once for any one + // cloud storage integration. Please do not turn it on for every integration test! + includeIncrementalSchemaSuite: false, + includeHLLSuite: false, + extendedEnv: 'export-bucket-s3-iam-roles' +}); From c8e7b93db51c2f5be4cb75c670037e64f8143606 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 11:11:26 -0400 Subject: [PATCH 02/16] fix handling --- .../src/SnowflakeDriver.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts index 2de7cce055ecf..dc925ac9ed4fd 100644 --- a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts +++ b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts @@ -753,7 +753,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { // Storage integration export flow takes precedence over direct auth if it is defined if (conf.integrationName) { optionsToExport.STORAGE_INTEGRATION = conf.integrationName; - } else { + } else if (conf.keyId && conf.secretKey) { optionsToExport.CREDENTIALS = `(AWS_KEY_ID = '${conf.keyId}' AWS_SECRET_KEY = '${conf.secretKey}')`; } } else if (bucketType === 'gcs') { @@ -793,14 +793,18 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { const { bucketName, path } = this.parseBucketUrl(this.config.exportBucket!.bucketName); const exportPrefix = path ? `${path}/${tableName}` : tableName; + const s3Config: any = { region }; + if (keyId && secretKey) { + // If access key and secret are provided, use them as credentials + // Otherwise, let the SDK use the default credential chain (IRSA, instance profile, etc.) + s3Config.credentials = { + accessKeyId: keyId, + secretAccessKey: secretKey, + }; + } + return this.extractUnloadedFilesFromS3( - { - credentials: { - accessKeyId: keyId, - secretAccessKey: secretKey, - }, - region, - }, + s3Config, bucketName, exportPrefix, ); From 9d2ee31f9fc7030da4a6746ceb20784780a3f0bf Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 11:31:52 -0400 Subject: [PATCH 03/16] lint --- packages/cubejs-testing-drivers/package.json | 1 + .../snowflake-export-bucket-s3-iam-roles-full.test.ts | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index 98b62043ea81c..a2b5317408b60 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -49,6 +49,7 @@ "snowflake-full": "yarn test-driver -i dist/test/snowflake-full.test.js", "snowflake-encrypted-pk-full": "yarn test-driver -i dist/test/snowflake-encrypted-pk-full.test.js", "snowflake-export-bucket-s3-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-full.test.js", + "snowflake-export-bucket-s3-iam-roles-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-iam-roles-full.test.js", "snowflake-export-bucket-s3-prefix-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-prefix-full.test.js", "snowflake-export-bucket-azure-full": "yarn test-driver -i dist/test/snowflake-export-bucket-azure-full.test.js", "snowflake-export-bucket-azure-prefix-full": "yarn test-driver -i dist/test/snowflake-export-bucket-azure-prefix-full.test.js", diff --git a/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts b/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts index 0a6649283d62e..7fddba8ac804c 100644 --- a/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts +++ b/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts @@ -1,9 +1,9 @@ import { testQueries } from '../src/tests/testQueries'; testQueries('snowflake', { - // NOTICE: It's enough to turn on this flag only once for any one - // cloud storage integration. Please do not turn it on for every integration test! - includeIncrementalSchemaSuite: false, - includeHLLSuite: false, - extendedEnv: 'export-bucket-s3-iam-roles' + // NOTICE: It's enough to turn on this flag only once for any one + // cloud storage integration. Please do not turn it on for every integration test! + includeIncrementalSchemaSuite: false, + includeHLLSuite: false, + extendedEnv: 'export-bucket-s3-iam-roles' }); From 4510ac7ac42966ee2df440a5be70ff3f817f4b7f Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 11:36:51 -0400 Subject: [PATCH 04/16] update docs --- .../configuration/data-sources/snowflake.mdx | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/snowflake.mdx b/docs/pages/product/configuration/data-sources/snowflake.mdx index 4a0d2639bd065..758f986f561a2 100644 --- a/docs/pages/product/configuration/data-sources/snowflake.mdx +++ b/docs/pages/product/configuration/data-sources/snowflake.mdx @@ -134,10 +134,11 @@ Storage][google-cloud-storage] for export bucket functionality. Ensure the AWS credentials are correctly configured in IAM to allow reads and -writes to the export bucket in S3 if you are not using storage integration. -If you are using storage integration then you still need to configure access keys -for Cube Store to be able to read from the export bucket. -It's possible to authenticate with IAM roles instead of access keys for Cube Store. +writes to the export bucket in S3. You can authenticate using: + +- **IAM user credentials** - Explicit AWS access keys and secrets +- **Storage integration** - Snowflake-managed AWS integration (may still require credentials for Cube Store) +- **IAM roles** - Use execution environment roles (IRSA, instance profiles) for both Snowflake and Cube Store @@ -163,7 +164,7 @@ CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET= CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ``` -Using Storage Integration to write to export bocket and IAM role to read from Cube Store: +Using Storage Integration to write to export bucket and IAM role to read from Cube Store: ```dotenv CUBEJS_DB_EXPORT_BUCKET_TYPE=s3 CUBEJS_DB_EXPORT_BUCKET=my.bucket.on.s3 @@ -171,6 +172,13 @@ CUBEJS_DB_EXPORT_INTEGRATION=aws_int CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ``` +Using IAM roles for both Snowflake and Cube Store (no credentials required): +```dotenv +CUBEJS_DB_EXPORT_BUCKET_TYPE=s3 +CUBEJS_DB_EXPORT_BUCKET=my.bucket.on.s3 +CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= +``` + #### Google Cloud Storage From e9da14b80a65640b80b2da5ccf23aeea0a5feb40 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 12:06:41 -0400 Subject: [PATCH 05/16] update update-all-snapshots-local --- packages/cubejs-testing-drivers/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index a2b5317408b60..9d15fd23e3a71 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -60,7 +60,7 @@ "redshift-core": "yarn test-driver -i dist/test/redshift-core.test.js", "redshift-full": "yarn test-driver -i dist/test/redshift-full.test.js", "redshift-export-bucket-s3-full": "yarn test-driver -i dist/test/redshift-export-bucket-s3-full.test.js", - "update-all-snapshots-local": "yarn run athena-export-bucket-s3-full --mode=local -u; yarn run bigquery-export-bucket-gcs-full --mode=local -u; yarn run clickhouse-full --mode=local -u; yarn run clickhouse-export-bucket-s3-full --mode=local -u; yarn run clickhouse-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-full --mode=local -u; yarn run mssql-full --mode=local -u; yarn run mysql-full --mode=local -u; yarn run postgres-full --mode=local -u; yarn run redshift-export-bucket-s3-full --mode=local -u; yarn run redshift-full --mode=local -u; yarn run snowflake-encrypted-pk-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-via-storage-integration-full --mode=local -u; yarn run snowflake-export-bucket-gcs-full --mode=local -u; yarn run snowflake-export-bucket-gcs-prefix-full --mode=local -u; yarn run snowflake-export-bucket-s3-full --mode=local -u; yarn run snowflake-export-bucket-s3-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-full --mode=local -u", + "update-all-snapshots-local": "yarn run athena-export-bucket-s3-full --mode=local -u; yarn run bigquery-export-bucket-gcs-full --mode=local -u; yarn run clickhouse-full --mode=local -u; yarn run clickhouse-export-bucket-s3-full --mode=local -u; yarn run clickhouse-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-full --mode=local -u; yarn run mssql-full --mode=local -u; yarn run mysql-full --mode=local -u; yarn run postgres-full --mode=local -u; yarn run redshift-export-bucket-s3-full --mode=local -u; yarn run redshift-full --mode=local -u; yarn run snowflake-encrypted-pk-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-via-storage-integration-full --mode=local -u; yarn run snowflake-export-bucket-gcs-full --mode=local -u; yarn run snowflake-export-bucket-gcs-prefix-full --mode=local -u; yarn run snowflake-export-bucket-s3-full --mode=local -u; yarn run snowflake-export-bucket-s3-iam-roles-full --mode=local -u; yarn run snowflake-export-bucket-s3-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-full --mode=local -u", "tst": "clear && yarn tsc && yarn bigquery-core" }, "files": [ From fc806503fdac61e5acee8be68c736385e975e2b3 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 13:28:35 -0400 Subject: [PATCH 06/16] add driver test to gh workflow --- .github/workflows/drivers-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index a125e7edae006..217d265933bef 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -225,6 +225,7 @@ jobs: snowflake snowflake-encrypted-pk snowflake-export-bucket-s3 + snowflake-export-bucket-s3-iam-roles snowflake-export-bucket-s3-prefix snowflake-export-bucket-azure snowflake-export-bucket-azure-prefix @@ -259,6 +260,7 @@ jobs: - snowflake - snowflake-encrypted-pk - snowflake-export-bucket-s3 + - snowflake-export-bucket-s3-iam-roles - snowflake-export-bucket-s3-prefix - snowflake-export-bucket-azure - snowflake-export-bucket-azure-prefix From fad29b1bc0519ff121b66bf397fa1252952fa493 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 13:52:21 -0400 Subject: [PATCH 07/16] address comments --- docs/pages/product/configuration/data-sources/snowflake.mdx | 2 +- packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/snowflake.mdx b/docs/pages/product/configuration/data-sources/snowflake.mdx index 758f986f561a2..25c2ba85ad2e0 100644 --- a/docs/pages/product/configuration/data-sources/snowflake.mdx +++ b/docs/pages/product/configuration/data-sources/snowflake.mdx @@ -137,7 +137,7 @@ Ensure the AWS credentials are correctly configured in IAM to allow reads and writes to the export bucket in S3. You can authenticate using: - **IAM user credentials** - Explicit AWS access keys and secrets -- **Storage integration** - Snowflake-managed AWS integration (may still require credentials for Cube Store) +- **Storage integration** - Snowflake-managed AWS integration (may still require credentials for exported file processing) - **IAM roles** - Use execution environment roles (IRSA, instance profiles) for both Snowflake and Cube Store diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts index dc925ac9ed4fd..45850fccc3cc8 100644 --- a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts +++ b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts @@ -24,6 +24,7 @@ import { formatToTimeZone } from 'date-fns-timezone'; import fs from 'fs/promises'; import crypto from 'crypto'; import { HydrationMap, HydrationStream } from './HydrationStream'; +import { S3ClientConfig } from '@aws-sdk/client-s3'; const SUPPORTED_BUCKET_TYPES = ['s3', 'gcs', 'azure']; @@ -793,7 +794,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { const { bucketName, path } = this.parseBucketUrl(this.config.exportBucket!.bucketName); const exportPrefix = path ? `${path}/${tableName}` : tableName; - const s3Config: any = { region }; + const s3Config: S3ClientConfig = { region }; if (keyId && secretKey) { // If access key and secret are provided, use them as credentials // Otherwise, let the SDK use the default credential chain (IRSA, instance profile, etc.) From f5367b52019e8bc24b28e9da3c5638c428da7db1 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 17:38:06 -0400 Subject: [PATCH 08/16] revert docs changes --- .../configuration/data-sources/snowflake.mdx | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/snowflake.mdx b/docs/pages/product/configuration/data-sources/snowflake.mdx index 25c2ba85ad2e0..5361802efdb83 100644 --- a/docs/pages/product/configuration/data-sources/snowflake.mdx +++ b/docs/pages/product/configuration/data-sources/snowflake.mdx @@ -134,11 +134,10 @@ Storage][google-cloud-storage] for export bucket functionality. Ensure the AWS credentials are correctly configured in IAM to allow reads and -writes to the export bucket in S3. You can authenticate using: - -- **IAM user credentials** - Explicit AWS access keys and secrets -- **Storage integration** - Snowflake-managed AWS integration (may still require credentials for exported file processing) -- **IAM roles** - Use execution environment roles (IRSA, instance profiles) for both Snowflake and Cube Store +writes to the export bucket in S3 if you are not using storage integration. +If you are using storage integration then you still need to configure access keys +for Cube Store to be able to read from the export bucket. +It's possible to authenticate with IAM roles instead of access keys for Cube Store. @@ -172,13 +171,6 @@ CUBEJS_DB_EXPORT_INTEGRATION=aws_int CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ``` -Using IAM roles for both Snowflake and Cube Store (no credentials required): -```dotenv -CUBEJS_DB_EXPORT_BUCKET_TYPE=s3 -CUBEJS_DB_EXPORT_BUCKET=my.bucket.on.s3 -CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= -``` - #### Google Cloud Storage From 44255aebf14b2208a067bcb260a9bf61d0c0e212 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Mon, 29 Sep 2025 18:02:02 -0400 Subject: [PATCH 09/16] update validation logic --- packages/cubejs-snowflake-driver/package.json | 1 + packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts | 7 ++----- packages/cubejs-testing-drivers/fixtures/snowflake.json | 3 ++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-snowflake-driver/package.json b/packages/cubejs-snowflake-driver/package.json index 604c2b0edbc97..0789ee8e9bdd5 100644 --- a/packages/cubejs-snowflake-driver/package.json +++ b/packages/cubejs-snowflake-driver/package.json @@ -25,6 +25,7 @@ "lint:fix": "eslint --fix src/* --ext .ts" }, "dependencies": { + "@aws-sdk/client-s3": "^3.726.0", "@cubejs-backend/base-driver": "1.3.75", "@cubejs-backend/shared": "1.3.75", "date-fns-timezone": "^0.1.4", diff --git a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts index 45850fccc3cc8..4b2b7df552c0c 100644 --- a/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts +++ b/packages/cubejs-snowflake-driver/src/SnowflakeDriver.ts @@ -23,8 +23,8 @@ import { import { formatToTimeZone } from 'date-fns-timezone'; import fs from 'fs/promises'; import crypto from 'crypto'; -import { HydrationMap, HydrationStream } from './HydrationStream'; import { S3ClientConfig } from '@aws-sdk/client-s3'; +import { HydrationMap, HydrationStream } from './HydrationStream'; const SUPPORTED_BUCKET_TYPES = ['s3', 'gcs', 'azure']; @@ -397,10 +397,7 @@ export class SnowflakeDriver extends BaseDriver implements DriverInterface { ): string[] { if (exportBucket.bucketType === 's3') { const s3Config = exportBucket as SnowflakeDriverExportAWS; - const hasCredentials = s3Config.keyId && s3Config.secretKey; - const hasIntegration = s3Config.integrationName; - - if (!hasCredentials && !hasIntegration) { + if (s3Config.integrationName) { return emptyKeys.filter(key => key !== 'keyId' && key !== 'secretKey'); } } diff --git a/packages/cubejs-testing-drivers/fixtures/snowflake.json b/packages/cubejs-testing-drivers/fixtures/snowflake.json index 83a6daa5080d6..f25e219ca3c3e 100644 --- a/packages/cubejs-testing-drivers/fixtures/snowflake.json +++ b/packages/cubejs-testing-drivers/fixtures/snowflake.json @@ -27,7 +27,8 @@ "environment": { "CUBEJS_DB_EXPORT_BUCKET_TYPE": "s3", "CUBEJS_DB_EXPORT_BUCKET": "snowflake-drivers-tests-preaggs", - "CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-west-1" + "CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-west-1", + "CUBEJS_DB_EXPORT_INTEGRATION": "aws_int" } } }, From 59aa8cb9e03f8dc443f87a9aebda015cb4ed72f2 Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Tue, 30 Sep 2025 08:40:55 -0400 Subject: [PATCH 10/16] update comments --- .../configuration/data-sources/snowflake.mdx | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/pages/product/configuration/data-sources/snowflake.mdx b/docs/pages/product/configuration/data-sources/snowflake.mdx index 5361802efdb83..c357294847e7a 100644 --- a/docs/pages/product/configuration/data-sources/snowflake.mdx +++ b/docs/pages/product/configuration/data-sources/snowflake.mdx @@ -133,15 +133,13 @@ Storage][google-cloud-storage] for export bucket functionality. -Ensure the AWS credentials are correctly configured in IAM to allow reads and -writes to the export bucket in S3 if you are not using storage integration. -If you are using storage integration then you still need to configure access keys -for Cube Store to be able to read from the export bucket. -It's possible to authenticate with IAM roles instead of access keys for Cube Store. +Ensure proper IAM privileges are configured for S3 bucket reads and writes, using either +storage integration or user credentials for Snowflake and either IAM roles/IRSA or user +credentials for Cube Store, with mixed configurations supported. -Using IAM user credentials: +Using IAM user credentials for both: ```dotenv CUBEJS_DB_EXPORT_BUCKET_TYPE=s3 @@ -151,8 +149,8 @@ CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET= CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ``` -[Using Storage Integration][snowflake-docs-aws-integration] to write to Export Bucket and -then Access Keys to read from Cube Store: +Using a [Storage Integration][snowflake-docs-aws-integration] to write to export buckets and +user credentials to read from Cube Store: ```dotenv CUBEJS_DB_EXPORT_BUCKET_TYPE=s3 @@ -163,7 +161,8 @@ CUBEJS_DB_EXPORT_BUCKET_AWS_SECRET= CUBEJS_DB_EXPORT_BUCKET_AWS_REGION= ``` -Using Storage Integration to write to export bucket and IAM role to read from Cube Store: +Using a Storage Integration to write to export bucket and IAM role/IRSA to read from Cube Store:** + ```dotenv CUBEJS_DB_EXPORT_BUCKET_TYPE=s3 CUBEJS_DB_EXPORT_BUCKET=my.bucket.on.s3 From 21f4d9b2a89d539d2db6b071546479ca3ab3b779 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 2 Oct 2025 13:04:07 +0300 Subject: [PATCH 11/16] fix storage integaration name --- packages/cubejs-testing-drivers/fixtures/snowflake.json | 4 ++-- packages/cubejs-testing-drivers/package.json | 4 ++-- ...-bucket-s3-via-storage-integration-iam-roles-full.test.ts} | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename packages/cubejs-testing-drivers/test/{snowflake-export-bucket-s3-iam-roles-full.test.ts => snowflake-export-bucket-s3-via-storage-integration-iam-roles-full.test.ts} (81%) diff --git a/packages/cubejs-testing-drivers/fixtures/snowflake.json b/packages/cubejs-testing-drivers/fixtures/snowflake.json index f25e219ca3c3e..3f0c9b9f9964e 100644 --- a/packages/cubejs-testing-drivers/fixtures/snowflake.json +++ b/packages/cubejs-testing-drivers/fixtures/snowflake.json @@ -22,13 +22,13 @@ } } }, - "export-bucket-s3-iam-roles": { + "export-bucket-s3-via-storage-integration-iam-roles": { "cube": { "environment": { "CUBEJS_DB_EXPORT_BUCKET_TYPE": "s3", "CUBEJS_DB_EXPORT_BUCKET": "snowflake-drivers-tests-preaggs", "CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-west-1", - "CUBEJS_DB_EXPORT_INTEGRATION": "aws_int" + "CUBEJS_DB_EXPORT_INTEGRATION": "drivers_tests_preaggs_s3" } } }, diff --git a/packages/cubejs-testing-drivers/package.json b/packages/cubejs-testing-drivers/package.json index c5d855709fbf1..1d5fe806f639c 100644 --- a/packages/cubejs-testing-drivers/package.json +++ b/packages/cubejs-testing-drivers/package.json @@ -49,7 +49,7 @@ "snowflake-full": "yarn test-driver -i dist/test/snowflake-full.test.js", "snowflake-encrypted-pk-full": "yarn test-driver -i dist/test/snowflake-encrypted-pk-full.test.js", "snowflake-export-bucket-s3-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-full.test.js", - "snowflake-export-bucket-s3-iam-roles-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-iam-roles-full.test.js", + "snowflake-export-bucket-s3-via-storage-integration-iam-roles-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-via-storage-integration-iam-roles-full.test.js", "snowflake-export-bucket-s3-prefix-full": "yarn test-driver -i dist/test/snowflake-export-bucket-s3-prefix-full.test.js", "snowflake-export-bucket-azure-full": "yarn test-driver -i dist/test/snowflake-export-bucket-azure-full.test.js", "snowflake-export-bucket-azure-prefix-full": "yarn test-driver -i dist/test/snowflake-export-bucket-azure-prefix-full.test.js", @@ -60,7 +60,7 @@ "redshift-core": "yarn test-driver -i dist/test/redshift-core.test.js", "redshift-full": "yarn test-driver -i dist/test/redshift-full.test.js", "redshift-export-bucket-s3-full": "yarn test-driver -i dist/test/redshift-export-bucket-s3-full.test.js", - "update-all-snapshots-local": "yarn run athena-export-bucket-s3-full --mode=local -u; yarn run bigquery-export-bucket-gcs-full --mode=local -u; yarn run clickhouse-full --mode=local -u; yarn run clickhouse-export-bucket-s3-full --mode=local -u; yarn run clickhouse-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-full --mode=local -u; yarn run mssql-full --mode=local -u; yarn run mysql-full --mode=local -u; yarn run postgres-full --mode=local -u; yarn run redshift-export-bucket-s3-full --mode=local -u; yarn run redshift-full --mode=local -u; yarn run snowflake-encrypted-pk-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-via-storage-integration-full --mode=local -u; yarn run snowflake-export-bucket-gcs-full --mode=local -u; yarn run snowflake-export-bucket-gcs-prefix-full --mode=local -u; yarn run snowflake-export-bucket-s3-full --mode=local -u; yarn run snowflake-export-bucket-s3-iam-roles-full --mode=local -u; yarn run snowflake-export-bucket-s3-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-full --mode=local -u", + "update-all-snapshots-local": "yarn run athena-export-bucket-s3-full --mode=local -u; yarn run bigquery-export-bucket-gcs-full --mode=local -u; yarn run clickhouse-full --mode=local -u; yarn run clickhouse-export-bucket-s3-full --mode=local -u; yarn run clickhouse-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-full --mode=local -u; yarn run databricks-jdbc-export-bucket-azure-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-full --mode=local -u; yarn run databricks-jdbc-export-bucket-gcs-prefix-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-full --mode=local -u; yarn run databricks-jdbc-export-bucket-s3-prefix-full --mode=local -u; yarn run databricks-jdbc-full --mode=local -u; yarn run mssql-full --mode=local -u; yarn run mysql-full --mode=local -u; yarn run postgres-full --mode=local -u; yarn run redshift-export-bucket-s3-full --mode=local -u; yarn run redshift-full --mode=local -u; yarn run snowflake-encrypted-pk-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-via-storage-integration-full --mode=local -u; yarn run snowflake-export-bucket-gcs-full --mode=local -u; yarn run snowflake-export-bucket-gcs-prefix-full --mode=local -u; yarn run snowflake-export-bucket-s3-full --mode=local -u; yarn run snowflake-export-bucket-s3-via-storage-integration-iam-roles-full --mode=local -u; yarn run snowflake-export-bucket-s3-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-prefix-full --mode=local -u; yarn run snowflake-export-bucket-azure-full --mode=local -u; yarn run snowflake-full --mode=local -u", "tst": "clear && yarn tsc && yarn bigquery-core" }, "files": [ diff --git a/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts b/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-via-storage-integration-iam-roles-full.test.ts similarity index 81% rename from packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts rename to packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-via-storage-integration-iam-roles-full.test.ts index 7fddba8ac804c..5e3d3a3e76d02 100644 --- a/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-iam-roles-full.test.ts +++ b/packages/cubejs-testing-drivers/test/snowflake-export-bucket-s3-via-storage-integration-iam-roles-full.test.ts @@ -5,5 +5,5 @@ testQueries('snowflake', { // cloud storage integration. Please do not turn it on for every integration test! includeIncrementalSchemaSuite: false, includeHLLSuite: false, - extendedEnv: 'export-bucket-s3-iam-roles' + extendedEnv: 'export-bucket-s3-via-storage-integration-iam-roles' }); From 0774c159ddee4917294af712fff3ff79cf9683fb Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 2 Oct 2025 13:14:30 +0300 Subject: [PATCH 12/16] Configure AWS credentials via IRSA --- .github/workflows/drivers-tests.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index 217d265933bef..fc161991e70af 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -225,7 +225,7 @@ jobs: snowflake snowflake-encrypted-pk snowflake-export-bucket-s3 - snowflake-export-bucket-s3-iam-roles + snowflake-export-bucket-s3-via-storage-integration-iam-roles snowflake-export-bucket-s3-prefix snowflake-export-bucket-azure snowflake-export-bucket-azure-prefix @@ -260,7 +260,7 @@ jobs: - snowflake - snowflake-encrypted-pk - snowflake-export-bucket-s3 - - snowflake-export-bucket-s3-iam-roles + - snowflake-export-bucket-s3-via-storage-integration-iam-roles - snowflake-export-bucket-s3-prefix - snowflake-export-bucket-azure - snowflake-export-bucket-azure-prefix @@ -340,6 +340,14 @@ jobs: gunzip image.tar.gz docker load -i image.tar + - name: Configure AWS credentials via IRSA + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DRIVERS_TESTS_AWS_ROLE_ARN_FOR_SNOWFLAKE }} + aws-region: us-west-1 + if: | + env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '' && matrix.database == 'snowflake-export-bucket-s3-via-storage-integration-iam-roles' + - name: Run tests uses: nick-fields/retry@v3 # It's enough to test for any one secret because they are set all at once or not set all From 541c34b59e3d6714ee0088fa035e7b5dd20d7e38 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 2 Oct 2025 13:47:59 +0300 Subject: [PATCH 13/16] add permissions --- .github/workflows/drivers-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index fc161991e70af..e59c34a8c9a62 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -204,6 +204,10 @@ jobs: tests: runs-on: ubuntu-24.04 + permissions: + id-token: write # Needed for OIDC+AWS + contents: read + timeout-minutes: 30 needs: [latest-tag-sha, build] if: (needs['latest-tag-sha'].outputs.sha != github.sha) @@ -345,6 +349,7 @@ jobs: with: role-to-assume: ${{ secrets.DRIVERS_TESTS_AWS_ROLE_ARN_FOR_SNOWFLAKE }} aws-region: us-west-1 + mask-aws-account-id: true if: | env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '' && matrix.database == 'snowflake-export-bucket-s3-via-storage-integration-iam-roles' From 3afae714516d21eb240a6fd46fff5e6c0900185e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 2 Oct 2025 13:57:40 +0300 Subject: [PATCH 14/16] code format --- .github/workflows/drivers-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/drivers-tests.yml b/.github/workflows/drivers-tests.yml index e59c34a8c9a62..c814f0efceca3 100644 --- a/.github/workflows/drivers-tests.yml +++ b/.github/workflows/drivers-tests.yml @@ -207,7 +207,7 @@ jobs: permissions: id-token: write # Needed for OIDC+AWS contents: read - + timeout-minutes: 30 needs: [latest-tag-sha, build] if: (needs['latest-tag-sha'].outputs.sha != github.sha) From 980e566f649ca27871e55bdba82cb4f3eb96a2cc Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 2 Oct 2025 17:01:11 +0300 Subject: [PATCH 15/16] uppercase storage integration --- packages/cubejs-testing-drivers/fixtures/snowflake.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-testing-drivers/fixtures/snowflake.json b/packages/cubejs-testing-drivers/fixtures/snowflake.json index 3f0c9b9f9964e..728d6c2a0085b 100644 --- a/packages/cubejs-testing-drivers/fixtures/snowflake.json +++ b/packages/cubejs-testing-drivers/fixtures/snowflake.json @@ -28,7 +28,7 @@ "CUBEJS_DB_EXPORT_BUCKET_TYPE": "s3", "CUBEJS_DB_EXPORT_BUCKET": "snowflake-drivers-tests-preaggs", "CUBEJS_DB_EXPORT_BUCKET_AWS_REGION": "us-west-1", - "CUBEJS_DB_EXPORT_INTEGRATION": "drivers_tests_preaggs_s3" + "CUBEJS_DB_EXPORT_INTEGRATION": "DRIVERS_TESTS_PREAGGS_S3" } } }, From 4a1dd70e11ea986aadc3e2715a270fc2d5e4e54c Mon Sep 17 00:00:00 2001 From: Matthew Orford Date: Thu, 2 Oct 2025 13:17:49 -0400 Subject: [PATCH 16/16] enable IRSA credential mounting for containerized AWS tests --- .../src/helpers/getComposePath.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/cubejs-testing-drivers/src/helpers/getComposePath.ts b/packages/cubejs-testing-drivers/src/helpers/getComposePath.ts index 5c0f895100251..42b2012b41b5c 100644 --- a/packages/cubejs-testing-drivers/src/helpers/getComposePath.ts +++ b/packages/cubejs-testing-drivers/src/helpers/getComposePath.ts @@ -27,6 +27,27 @@ export function getComposePath(type: string, fixture: Fixture, isLocal: boolean) './package.json:/cube/conf/package.json', './model/ecommerce.yaml:/cube/conf/model/ecommerce.yaml', ]; + + // Add AWS credential mounting for IRSA-enabled tests + if (process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY && process.env.AWS_SESSION_TOKEN) { + const awsCredentialsDir = path.resolve(_path, '.aws'); + fs.ensureDirSync(awsCredentialsDir); + + const credentialsContent = `[default] +aws_access_key_id = ${process.env.AWS_ACCESS_KEY_ID} +aws_secret_access_key = ${process.env.AWS_SECRET_ACCESS_KEY} +aws_session_token = ${process.env.AWS_SESSION_TOKEN} +`; + + const configContent = `[default] +region = ${process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-west-1'} +`; + + fs.writeFileSync(path.resolve(awsCredentialsDir, 'credentials'), credentialsContent); + fs.writeFileSync(path.resolve(awsCredentialsDir, 'config'), configContent); + + volumes.push('./.aws:/root/.aws:ro'); + } const compose: any = { version: '2.2', services: { @@ -46,6 +67,9 @@ export function getComposePath(type: string, fixture: Fixture, isLocal: boolean) image: `cubejs/cubestore:${process.arch === 'arm64' ? 'arm64v8' : 'latest'}`, ports: ['3030'], restart: 'always', + ...(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY && process.env.AWS_SESSION_TOKEN ? { + volumes: ['./.aws:/root/.aws:ro'] + } : {}) } } };