diff --git a/README.md b/README.md index db14c40..4172d8e 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ The following options are available: `sql.db.''.password` : The database connection password. +For information on using secrets with database credentials, see [docs/secrets.md](docs/secrets.md). + ## Dataflow Operators This plugin provides the following dataflow operators for querying from and inserting into database tables. diff --git a/changelog.txt b/changelog.txt index ff59438..f84daa5 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,5 +1,10 @@ NF-SQLDB CHANGE-LOG =================== +0.7.1 - 21 Aug 2025 +- Fix unresolved secrets detection to prevent silent fallback to default credentials +- Add comprehensive secrets documentation and troubleshooting guide +- Improve error messages for unknown database configurations + 0.7.0 - 28 May 2025 - Add sqlExecutor and other minor improvements [072ae039] diff --git a/docs/secrets.md b/docs/secrets.md new file mode 100644 index 0000000..7175c15 --- /dev/null +++ b/docs/secrets.md @@ -0,0 +1,55 @@ +# Using Secrets for Database Credentials + +For production deployments, it's recommended to use Nextflow secrets instead of hardcoding credentials in configuration files. This is especially important when working with cloud databases like AWS Athena. + +## Configuration with Secrets + +When using [Nextflow secrets](https://www.nextflow.io/docs/latest/secrets.html) (available in Nextflow 25.04.0+), you can reference workspace or user-level secrets in your database configuration: + +```groovy +sql { + db { + athena { + url = 'jdbc:awsathena://AwsRegion=us-east-1;S3OutputLocation=s3://bucket;Workgroup=workgroup' + user = secrets.ATHENA_USER + password = secrets.ATHENA_PASSWORD + driver = 'com.simba.athena.jdbc.Driver' + } + } +} +``` + +## Setting Up Secrets in Seqera Platform + +1. **Workspace Secrets**: Navigate to your workspace → Secrets → Add secret +2. **User Secrets**: Navigate to Your profile → Secrets → Add secret +3. Create secrets with names matching your configuration (e.g., `ATHENA_USER`, `ATHENA_PASSWORD`) + +## Troubleshooting Secrets Issues + +If you encounter authentication errors like "Missing credentials error", verify: + +- **Secret Names**: Ensure secret names in your configuration match exactly (case-sensitive) +- **Permissions**: Verify you have access to workspace secrets or have defined user-level secrets +- **Nextflow Version**: Secrets require Nextflow >=25.04.0 +- **Secret Values**: Ensure secrets contain valid credentials (no empty values) + +Common error patterns: +- `user=sa; password=null` indicates secrets were not resolved +- `Unresolved secret detected` means secret names don't match or aren't accessible + +## Local Development + +For local testing, you can use the Nextflow secrets command: + +```bash +# Set secrets locally +nextflow secrets set ATHENA_USER "your-username" +nextflow secrets set ATHENA_PASSWORD "your-password" + +# List secrets +nextflow secrets list + +# Run pipeline with secrets +nextflow run your-pipeline.nf +``` \ No newline at end of file diff --git a/plugins/nf-sqldb/src/main/nextflow/sql/ChannelSqlExtension.groovy b/plugins/nf-sqldb/src/main/nextflow/sql/ChannelSqlExtension.groovy index 987602c..8244fd2 100644 --- a/plugins/nf-sqldb/src/main/nextflow/sql/ChannelSqlExtension.groovy +++ b/plugins/nf-sqldb/src/main/nextflow/sql/ChannelSqlExtension.groovy @@ -106,11 +106,9 @@ class ChannelSqlExtension extends PluginExtensionPoint { final dataSource = config.getDataSource(dsName) if( dataSource==null ) { def msg = "Unknown db name: $dsName" - def choices = config.getDataSourceNames().closest(dsName) ?: config.getDataSourceNames() - if( choices?.size() == 1 ) - msg += " - Did you mean: ${choices.get(0)}?" - else if( choices ) - msg += " - Did you mean any of these?\n" + choices.collect { " $it"}.join('\n') + '\n' + def choices = config.getDataSourceNames() + if( choices ) + msg += " - Available databases: " + choices.join(', ') throw new IllegalArgumentException(msg) } return dataSource diff --git a/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy b/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy index 1af3742..7a93210 100644 --- a/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy +++ b/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy @@ -43,21 +43,53 @@ class SqlDataSource { SqlDataSource(Map opts) { this.url = opts.url ?: DEFAULT_URL this.driver = opts.driver ?: urlToDriver(url) ?: DEFAULT_DRIVER - this.user = opts.user ?: DEFAULT_USER - this.password = opts.password + this.user = resolveCredential(opts.user, 'user') ?: DEFAULT_USER + this.password = resolveCredential(opts.password, 'password') } SqlDataSource(Map opts, SqlDataSource fallback) { this.url = opts.url ?: fallback.url ?: DEFAULT_URL this.driver = opts.driver ?: urlToDriver(url) ?: fallback.driver ?: DEFAULT_DRIVER - this.user = opts.user ?: fallback.user ?: DEFAULT_USER - this.password = opts.password ?: fallback.password + this.user = resolveCredential(opts.user, 'user') ?: fallback.user ?: DEFAULT_USER + this.password = resolveCredential(opts.password, 'password') ?: fallback.password } protected String urlToDriver(String url) { DriverRegistry.DEFAULT.urlToDriver(url) } + /** + * Resolves a credential value, checking for unresolved secrets and providing appropriate error handling + * + * @param value The credential value from configuration + * @param credType The type of credential ('user' or 'password') for error messages + * @return The resolved credential value, or null if not provided + * @throws IllegalArgumentException if an unresolved secret is detected + */ + protected String resolveCredential(Object value, String credType) { + if (value == null) { + return null + } + + String stringValue = value.toString() + + // Check for unresolved secrets (patterns like 'secrets.ATHENA_USER' or similar) + if (stringValue.startsWith('secrets.') || stringValue.contains('secret') && stringValue.contains('[') && stringValue.contains(']')) { + throw new IllegalArgumentException( + "Unresolved secret detected for $credType: '$stringValue'. " + + "This typically indicates that workspace secrets are not properly configured or accessible. " + + "Please verify that:\n" + + "1. The secret is defined in your workspace/user secrets\n" + + "2. The secret name matches exactly (case-sensitive)\n" + + "3. You have proper permissions to access the secret\n" + + "4. The Nextflow version supports secrets integration (>=25.04.0)\n" + + "See: https://www.nextflow.io/docs/latest/secrets.html" + ) + } + + return stringValue.isEmpty() ? null : stringValue + } + Map toMap() { final result = new HashMap(10) if( url ) diff --git a/plugins/nf-sqldb/src/test/nextflow/sql/ChannelSqlExtensionTest.groovy b/plugins/nf-sqldb/src/test/nextflow/sql/ChannelSqlExtensionTest.groovy index ef4590b..28981d2 100644 --- a/plugins/nf-sqldb/src/test/nextflow/sql/ChannelSqlExtensionTest.groovy +++ b/plugins/nf-sqldb/src/test/nextflow/sql/ChannelSqlExtensionTest.groovy @@ -95,4 +95,34 @@ class ChannelSqlExtensionTest extends Specification { rows.alpha == ['x1','y2','z3'] } + def 'should error on unresolved secrets' () { + given: + def session = Mock(Session) { + getConfig() >> [sql: [db: [test: [user: 'secrets.ATHENA_USER']]]] + } + + when: + new ChannelSqlExtension().init(session) + + then: + thrown(IllegalArgumentException) + } + + def 'should handle unknown database' () { + given: + def session = Mock(Session) { + getConfig() >> [sql: [db: [default: [url: 'jdbc:h2:mem:'], postgres: [url: 'jdbc:postgresql:']]]] + } + def sqlExtension = new ChannelSqlExtension() + sqlExtension.init(session) + + when: + sqlExtension.fromQuery([db: 'invalid'], 'select * from table') + + then: + def e = thrown(IllegalArgumentException) + e.message.contains("Unknown db name: invalid") + e.message.contains("Available databases:") + } + } diff --git a/plugins/nf-sqldb/src/test/nextflow/sql/SqlDslTest.groovy b/plugins/nf-sqldb/src/test/nextflow/sql/SqlDslTest.groovy index 1ad3a38..fc5572c 100644 --- a/plugins/nf-sqldb/src/test/nextflow/sql/SqlDslTest.groovy +++ b/plugins/nf-sqldb/src/test/nextflow/sql/SqlDslTest.groovy @@ -209,6 +209,12 @@ class SqlDslTest extends Dsl2Spec { @Requires({System.getenv('NF_SQLDB_TEST_ATHENA_REGION')}) @Requires({System.getenv('NF_SQLDB_ATHENA_TEST_S3_BUCKET')}) @IgnoreIf({ System.getenv('NXF_SMOKE') }) + @Requires({ + System.getenv('NF_SQLDB_TEST_ATHENA_USERNAME') && + System.getenv('NF_SQLDB_TEST_ATHENA_PASSWORD') && + System.getenv('NF_SQLDB_TEST_ATHENA_REGION') && + System.getenv('NF_SQLDB_ATHENA_TEST_S3_BUCKET') + }) @Timeout(60) def 'should perform a query for AWS Athena and create a channel'() { given: diff --git a/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy b/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy index 2975c70..95d165e 100644 --- a/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy +++ b/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy @@ -132,4 +132,30 @@ class SqlDataSourceTest extends Specification { ds1.hashCode() == ds2.hashCode() ds1.hashCode() != ds3.hashCode() } + + def 'should detect unresolved secrets' () { + when: + new SqlDataSource([user: pattern]) + then: + def e = thrown(IllegalArgumentException) + e.message.contains("Unresolved secret detected") + e.message.contains("workspace secrets are not properly configured") + + where: + pattern << ['secrets.ATHENA_USER', '[secret]'] + } + + def 'should handle various credential inputs' () { + when: + def ds = new SqlDataSource([user: userInput, password: passInput]) + then: + ds.user == expectedUser + ds.password == expectedPass + + where: + userInput | passInput | expectedUser | expectedPass + 'validuser' | 'validpass' | 'validuser' | 'validpass' + null | null | SqlDataSource.DEFAULT_USER| null + '' | '' | SqlDataSource.DEFAULT_USER| null + } }