Move shared Platform code to separate modules

CLAUDE.md

@@ -38,7 +38,10 @@ The project follows a modular architecture with a plugin-based system for cloud
 
 ### Core Modules (`modules/`)
 - **nextflow**: Main application module with core workflow engine, CLI, AST transformations, executors
-- **nf-commons**: Shared utilities, plugin system infrastructure, extension methods
+- **nf-commons**: Shared utilities, plugin system infrastructure, extension methods, common exceptions
+- **nf-config**: Configuration parsing and management (ConfigBuilder, ConfigParser, Manifest, WorkflowConfig)
+- **nf-scm**: Source code management (AssetManager, repository providers for GitHub/GitLab/Bitbucket/Azure/Gitea)
+- **nf-secrets**: Secrets management (SecretsProvider, SecretsLoader, integration with external secret stores)
 - **nf-httpfs**: HTTP filesystem support and custom providers
 - **nf-lang**: Language parsing, ANTLR grammars, AST implementation
 - **nf-lineage**: Data lineage tracking and workflow execution history

build.gradle

@@ -325,7 +325,7 @@ task compile {
 
 def getRuntimeConfigs() {
     def names = subprojects
-            .findAll { prj -> prj.name in ['nextflow','nf-commons','nf-httpfs','nf-lang','nf-lineage'] }
+            .findAll { prj -> prj.name in ['nextflow','nf-commons','nf-scm','nf-config','nf-secrets','nf-httpfs','nf-lang','nf-lineage'] }
             .collect { it.name }
 
     FileCollection result = null
@@ -356,14 +356,14 @@ task exportClasspath {
         def libs = []
 
         // Resolve configurations during provider evaluation (not ideal but functional)
-        ['nextflow','nf-commons','nf-httpfs','nf-lang','nf-lineage'].each { moduleName ->
+        ['nextflow','nf-commons','nf-scm','nf-config','nf-secrets','nf-httpfs','nf-lang','nf-lineage'].each { moduleName ->
             def moduleProject = project(":$moduleName")
             def cfg = moduleProject.configurations.getByName('runtimeClasspath')
             libs.addAll(cfg.files.collect { it.canonicalPath })
         }
 
         // Add module jars
-        ['nextflow','nf-commons','nf-httpfs','nf-lang','nf-lineage'].each {
+        ['nextflow','nf-commons','nf-scm','nf-config','nf-secrets','nf-httpfs','nf-lang','nf-lineage'].each {
             libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath
         }
 
@@ -387,7 +387,7 @@ ext.nexusEmail = project.findProperty('nexusEmail')
 // `signing.keyId` property needs to be defined in the `gradle.properties` file
 ext.enableSignArchives = project.findProperty('signing.keyId')
 
-ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-httpfs', ':nf-lang', ':nf-lineage' )
+ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-scm', ':nf-config', ':nf-secrets', ':nf-httpfs', ':nf-lang', ':nf-lineage' )
 
 configure(coreProjects) {
     group = 'io.nextflow'

modules/nextflow/build.gradle

@@ -24,6 +24,9 @@ configurations {
 dependencies {
     api(project(':nf-commons'))
     api(project(':nf-httpfs'))
+    api(project(':nf-scm'))
+    api(project(':nf-secrets'))
+    api(project(':nf-config'))
     api(project(':nf-lang'))
     api "org.apache.groovy:groovy:4.0.29"
     api "org.apache.groovy:groovy-nio:4.0.29"
@@ -44,7 +47,6 @@ dependencies {
     api "com.beust:jcommander:1.35"
     api("com.esotericsoftware.kryo:kryo:2.24.0") { exclude group: 'com.esotericsoftware.minlog', module: 'minlog' }
     api('org.iq80.leveldb:leveldb:0.12')
-    api('org.eclipse.jgit:org.eclipse.jgit:7.1.1.202505221757-r')
     api ('javax.activation:activation:1.1.1')
     api ('javax.mail:mail:1.4.7')
     api ('org.yaml:snakeyaml:2.2')

modules/nextflow/src/main/groovy/nextflow/NF.groovy

@@ -33,11 +33,11 @@ class NF {
     }
 
     static String getSyntaxParserVersion() {
-        return SysEnv.get('NXF_SYNTAX_PARSER', 'v1')
+        return Const.getSyntaxParserVersion()
     }
 
     static boolean isSyntaxParserV2() {
-        return getSyntaxParserVersion() == 'v2'
+        return Const.isSyntaxParserV2()
     }
 
     static void init() {

modules/nextflow/src/main/groovy/nextflow/cli/HubOptions.groovy → modules/nextflow/src/main/groovy/nextflow/cli/CliHubOptions.groovy

@@ -18,14 +18,16 @@ package nextflow.cli
 
 import com.beust.jcommander.Parameter
 import groovy.transform.CompileStatic
+import nextflow.scm.HubOptions
+
 /**
   * Defines the command line parameters for command that need to interact with a pipeline service hub i.e. GitHub or BitBucket
   *
   * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
   */
 
 @CompileStatic
-trait HubOptions {
+trait CliHubOptions implements HubOptions {
 
     @Parameter(names=['-hub'], description = "Service hub where the project is hosted")
     String hubProvider

modules/nextflow/src/main/groovy/nextflow/cli/CliOptions.groovy

@@ -21,6 +21,7 @@ import com.beust.jcommander.Parameter
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.SysEnv
+import nextflow.config.ConfigCliOptions
 import nextflow.exception.AbortOperationException
 import org.fusesource.jansi.Ansi
 
@@ -31,7 +32,7 @@ import org.fusesource.jansi.Ansi
  */
 @Slf4j
 @CompileStatic
-class CliOptions {
+class CliOptions implements ConfigCliOptions{
 
     /**
      * The packages to debug

modules/nextflow/src/main/groovy/nextflow/cli/CmdClean.groovy

@@ -108,7 +108,7 @@ class CmdClean extends CmdBase implements CacheBase {
             final builder = new ConfigBuilder()
                     .setShowClosures(true)
                     .showMissingVariables(true)
-                    .setOptions(launcher.options)
+                    .setCliOptions(launcher.options)
                     .setBaseDir(Paths.get('.'))
 
             final config = builder.buildConfigObject()

modules/nextflow/src/main/groovy/nextflow/cli/CmdClone.groovy

@@ -30,7 +30,7 @@ import nextflow.scm.AssetManager
 @Slf4j
 @CompileStatic
 @Parameters(commandDescription = "Clone a project into a folder")
-class CmdClone extends CmdBase implements HubOptions {
+class CmdClone extends CmdBase implements CliHubOptions {
 
     static final public NAME = 'clone'
 

modules/nextflow/src/main/groovy/nextflow/cli/CmdConfig.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.cli
 
+import nextflow.Const
+
 import java.nio.file.Path
 import java.nio.file.Paths
 
@@ -114,14 +116,15 @@ class CmdConfig extends CmdBase {
                 .setShowClosures(true)
                 .setStripSecrets(true)
                 .showMissingVariables(true)
-                .setOptions(launcher.options)
+                .setCliOptions(launcher.options)
                 .setBaseDir(base)
-                .setCmdConfig(this)
+                .setShowAllProfiles(this.showAllProfiles)
+                .setProfile(this.profile)
 
         final config = builder.buildConfigObject()
 
         // -- validate config options
-        if( NF.isSyntaxParserV2() ) {
+        if( Const.isSyntaxParserV2() ) {
             Plugins.load(config)
             new ConfigValidator().validate(config)
         }

modules/nextflow/src/main/groovy/nextflow/cli/CmdFs.groovy

@@ -203,7 +203,7 @@ class CmdFs extends CmdBase implements UsageAware {
     private Session createSession() {
         // create the config
         final config = new ConfigBuilder()
-                .setOptions(getLauncher().getOptions())
+                .setCliOptions(getLauncher().getOptions())
                 .setBaseDir(Paths.get('.'))
                 .build()
 

modules/nextflow/src/main/groovy/nextflow/cli/CmdLineage.groovy

@@ -88,7 +88,7 @@ class CmdLineage extends CmdBase implements UsageAware {
         Plugins.init()
         // load the config
         this.config = new ConfigBuilder()
-            .setOptions(launcher.options)
+            .setCliOptions(launcher.options)
             .setBaseDir(Paths.get('.'))
             .build()
         // init plugins

modules/nextflow/src/main/groovy/nextflow/cli/CmdNode.groovy

@@ -21,6 +21,7 @@ import com.beust.jcommander.Parameters
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.config.ConfigBuilder
+import nextflow.config.ConfigNodeOptions
 import nextflow.daemon.DaemonLauncher
 import nextflow.plugin.Plugins
 import nextflow.util.ServiceName
@@ -32,7 +33,7 @@ import nextflow.util.ServiceDiscover
 @Slf4j
 @CompileStatic
 @Parameters
-class CmdNode extends CmdBase {
+class CmdNode extends CmdBase implements ConfigNodeOptions {
 
     static final public NAME = 'node'
 
@@ -66,8 +67,8 @@ class CmdNode extends CmdBase {
 
         // create the config object
         def config = new ConfigBuilder()
-                            .setOptions(launcher.options)
-                            .setCmdNode(this)
+                            .setCliOptions(launcher.options)
+                            .setNodeOptions(this)
                             .build()
 
         DaemonLauncher instance

modules/nextflow/src/main/groovy/nextflow/cli/CmdPull.groovy

@@ -31,7 +31,7 @@ import nextflow.util.TestOnly
 @Slf4j
 @CompileStatic
 @Parameters(commandDescription = "Download or update a project")
-class CmdPull extends CmdBase implements HubOptions {
+class CmdPull extends CmdBase implements CliHubOptions {
 
     static final public NAME = 'pull'
 

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -16,6 +16,8 @@
 
 package nextflow.cli
 
+import nextflow.config.ConfigCliOptions
+
 import static org.fusesource.jansi.Ansi.*
 
 import java.nio.file.NoSuchFileException
@@ -32,11 +34,13 @@ import groovy.transform.Memoized
 import groovy.util.logging.Slf4j
 import groovyx.gpars.GParsConfig
 import nextflow.BuildInfo
+import nextflow.Const
 import nextflow.NF
 import nextflow.NextflowMeta
 import nextflow.SysEnv
 import nextflow.config.ConfigBuilder
 import nextflow.config.ConfigMap
+import nextflow.config.ConfigRunOptions
 import nextflow.config.ConfigValidator
 import nextflow.config.Manifest
 import nextflow.exception.AbortOperationException
@@ -62,7 +66,7 @@ import org.yaml.snakeyaml.Yaml
 @Slf4j
 @CompileStatic
 @Parameters(commandDescription = "Execute a pipeline project")
-class CmdRun extends CmdBase implements HubOptions {
+class CmdRun extends CmdBase implements CliHubOptions, ConfigRunOptions{
 
     static final public Pattern RUN_NAME_PATTERN = Pattern.compile(/^[a-z](?:[a-z\d]|[-_](?=[a-z\d])){0,79}$/, Pattern.CASE_INSENSITIVE)
 
@@ -354,8 +358,8 @@ class CmdRun extends CmdBase implements HubOptions {
         // -- PHASE 1: Load config with mock secrets provider
         final secretsProvider = new EmptySecretProvider()
         ConfigBuilder builder = new ConfigBuilder()
-            .setOptions(launcher.options)
-            .setCmdRun(this)
+            .setCliOptions(launcher.options)
+            .setRunOptions(this as ConfigRunOptions)
             .setBaseDir(scriptFile.parent)
             .setCliParams(cliParams)
             .setSecretsProvider(secretsProvider)  // Mock provider returns empty strings
@@ -382,8 +386,8 @@ class CmdRun extends CmdBase implements HubOptions {
         if( secretsProvider.usedSecrets() ) {
             log.debug "Config file used secrets -- reloading config with secrets provider"
             builder = new ConfigBuilder()
-                .setOptions(launcher.options)
-                .setCmdRun(this)
+                .setCliOptions(launcher.options)
+                .setRunOptions(this)
                 .setBaseDir(scriptFile.parent)
                 .setCliParams(cliParams)
                 // No .setSecretsProvider() - uses real secrets system now
@@ -395,7 +399,7 @@ class CmdRun extends CmdBase implements HubOptions {
         launchInfo(config, scriptFile)
 
         // -- validate config options
-        if( NF.isSyntaxParserV2() )
+        if( Const.isSyntaxParserV2() )
             new ConfigValidator().validate(config)
 
         // -- create a new runner instance
@@ -799,7 +803,7 @@ class CmdRun extends CmdBase implements HubOptions {
     }
 
     static protected parseParamValue(String str) {
-        if ( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || NF.isSyntaxParserV2() )
+        if ( SysEnv.get('NXF_DISABLE_PARAMS_TYPE_DETECTION') || Const.isSyntaxParserV2() )
             return str
 
         if ( str == null ) return null
@@ -880,4 +884,25 @@ class CmdRun extends CmdBase implements HubOptions {
         }
     }
 
+    ConfigCliOptions getConfigCliOptions(){
+        return launcher.options as ConfigCliOptions
+    }
+
+    String getNormalizeResumeId( String previousId ) {
+        String uniqueId = resume ?: previousId
+        if( !uniqueId )
+            return null
+        if( uniqueId == 'last' || uniqueId == 'true' ) {
+            if( HistoryFile.disabled() )
+                throw new AbortOperationException("The resume session id should be specified via `-resume` option when history file tracking is disabled")
+            uniqueId = HistoryFile.DEFAULT.getLast()?.sessionId
+
+            if( !uniqueId ) {
+                log.warn "It appears you have never run this project before -- Option `-resume` is ignored"
+            }
+        }
+
+        return uniqueId
+    }
+
 }

modules/nextflow/src/main/groovy/nextflow/cli/PluginAbstractExec.groovy

@@ -51,7 +51,7 @@ trait PluginAbstractExec implements PluginExecAware {
         this.launcher = launcher1
         // create the config
         final config = new ConfigBuilder()
-                .setOptions(launcher1.options)
+                .setCliOptions(launcher1.options)
                 .setBaseDir(Paths.get('.'))
                 .build()
 

modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy

@@ -16,7 +16,7 @@
 
 package nextflow.script
 
-import static nextflow.util.ConfigHelper.*
+import static nextflow.util.ConfigHelper.parseValue
 
 import java.nio.file.Path
 

modules/nf-commons/build.gradle

@@ -39,6 +39,7 @@ dependencies {
     // patch gson dependency required by pf4j
     api 'com.google.code.gson:gson:2.13.1'
     api 'io.seqera:npr-api:0.6.1'
+    api 'jline:jline:2.9'
 
     /* testImplementation inherited from top gradle build file */
     testImplementation(testFixtures(project(":nextflow")))

modules/nf-commons/src/main/nextflow/Const.groovy

@@ -70,6 +70,14 @@ class Const {
         return Path.of(SysEnv.get('NXF_CACHE_DIR', '.nextflow'))
     }
 
+    static String getSyntaxParserVersion() {
+        return SysEnv.get('NXF_SYNTAX_PARSER', 'v1')
+    }
+
+    static boolean isSyntaxParserV2() {
+        return getSyntaxParserVersion() == 'v2'
+    }
+
     static public final String ROLE_WORKER = 'worker'
 
     static public final String ROLE_MASTER = 'master'

modules/nf-config/build.gradle

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2013-2024, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+apply plugin: 'groovy'
+
+sourceSets {
+    main.java.srcDirs = []
+    main.groovy.srcDirs = ['src/main']
+    main.resources.srcDirs = ['src/resources']
+    test.groovy.srcDirs = ['src/test']
+    test.java.srcDirs = []
+    test.resources.srcDirs = []
+}
+
+
+dependencies {
+    api project(':nf-commons')
+    api project(':nf-secrets')
+    api ('org.apache.groovy:groovy:4.0.29')
+    api ('org.apache.groovy:groovy-nio:4.0.29')
+    api ('org.apache.groovy:groovy-json:4.0.29')
+    api ('ch.artecat.grengine:grengine:3.0.2') { exclude group: 'org.codehaus.groovy' }
+    api ('org.yaml:snakeyaml:2.2')
+
+    /* testImplementation inherited from top gradle build file */
+    testImplementation "org.apache.groovy:groovy-json:4.0.29" // needed by wiremock
+    testImplementation(testFixtures(project(":nextflow")))
+}
+