Skip to content

Commit 2fa01e8

Browse files
jorgeejorgee
authored
Remote module inclusion without @ (#6898)
1 parent 72ec5d9 commit 2fa01e8

29 files changed

+339
-357
lines changed

‎modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInfo.groovy‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -90,8 +90,7 @@ class CmdModuleInfo extends CmdBase {
9090
throw new AbortOperationException("Incorrect number of arguments")
9191
}
9292

93-
def moduleRef = '@' + args[0]
94-
def reference = ModuleReference.parse(moduleRef)
93+
def reference = ModuleReference.parse(args[0])
9594

9695
// Get config
9796
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
@@ -117,10 +116,10 @@ class CmdModuleInfo extends CmdBase {
117116
log.warn "Failed to fetch metadata from registry: ${e.message}"
118117
}
119118
if( !release ) {
120-
throw new AbortOperationException("No release information available for ${reference.nameWithoutPrefix}")
119+
throw new AbortOperationException("No release information available for ${reference}")
121120
}
122121
if( !release.metadata ) {
123-
log.info("No metadata found for $reference.nameWithoutPrefix ${release.version ? "($release.version)" : ''}")
122+
log.info("No metadata found for $reference ${release.version ? "($release.version)" : ''}")
124123
}
125124
def moduleUrl = buildModuleUrl(registryConfig.url, reference, release.version)
126125
if( !output || output == 'text' ) {
@@ -135,7 +134,7 @@ class CmdModuleInfo extends CmdBase {
135134
private void printFormattedInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) {
136135
ModuleMetadata metadata = release.metadata
137136
println ""
138-
println "Module: ${reference.nameWithoutPrefix}"
137+
println "Module: ${reference}"
139138
println "Version: ${release.version}"
140139
println "URL: ${moduleUrl}"
141140
println "Description: ${metadata.description ?: release.description ?: 'N/A'}"
@@ -218,7 +217,7 @@ class CmdModuleInfo extends CmdBase {
218217

219218
private List<String> generateUsageTemplate(ModuleReference reference, ModuleMetadata metadata) {
220219
def template = new ArrayList<String>()
221-
template.add("nextflow module run ${reference.nameWithoutPrefix}".toString())
220+
template.add("nextflow module run ${reference}".toString())
222221
if( version )
223222
template.add(" -version $version".toString())
224223

@@ -281,7 +280,7 @@ class CmdModuleInfo extends CmdBase {
281280
private void printJsonInfo(ModuleReference reference, ModuleRelease release, String moduleUrl) {
282281
def metadata = release?.metadata
283282
def info = [
284-
name : reference.nameWithoutPrefix,
283+
name : reference.toString(),
285284
fullName : reference.fullName,
286285
version : release.version,
287286
url : moduleUrl,

‎modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleInstall.groovy‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,7 @@ class CmdModuleInstall extends CmdBase {
7070
throw new AbortOperationException("Incorrect number of arguments")
7171
}
7272

73-
def moduleRef = '@' + args[0]
74-
75-
def reference = ModuleReference.parse(moduleRef)
73+
def reference = ModuleReference.parse(args[0])
7674

7775
// Get config
7876
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
@@ -96,7 +94,7 @@ class CmdModuleInstall extends CmdBase {
9694
def installedVersion = version ?: resolver.resolveVersion(reference)
9795
specFile.addModuleEntry(reference.fullName, installedVersion)
9896

99-
println "Module ${reference.nameWithoutPrefix}@${installedVersion} installed and configured successfully"
97+
println "Module ${reference}@${installedVersion} installed and configured successfully"
10098
}
10199
catch( AbortOperationException e ) {
102100
throw e

‎modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleList.groovy‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,15 +111,15 @@ class CmdModuleList extends CmdBase {
111111

112112
installed.each { module ->
113113
def status = getStatusString(module.integrity)
114-
println "${module.reference.nameWithoutPrefix.padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}"
114+
println "${module.reference.toString().padRight(40)}${(module.installedVersion ?: 'unknown').padRight(15)}${status}"
115115
}
116116
println ""
117117
}
118118

119119
private void printJsonList(List<InstalledModule> installed) {
120120
def modules = installed.collect { module ->
121121
[
122-
name : module.reference.nameWithoutPrefix,
122+
name : module.reference.toString(),
123123
version : module.installedVersion ?: 'unknown',
124124
integrity: module.integrity.toString(),
125125
directory: module.directory.toString()
@@ -136,8 +136,8 @@ class CmdModuleList extends CmdBase {
136136
return 'OK'
137137
case ModuleIntegrity.MODIFIED:
138138
return 'MODIFIED'
139-
case ModuleIntegrity.MISSING_CHECKSUM:
140-
return 'NO CHECKSUM'
139+
case ModuleIntegrity.NO_REMOTE_MODULE:
140+
return 'LOCAL'
141141
case ModuleIntegrity.CORRUPTED:
142142
return 'CORRUPTED'
143143
default:

‎modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModulePublish.groovy‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import nextflow.cli.CmdBase
2525
import nextflow.config.ConfigBuilder
2626
import nextflow.config.RegistryConfig
2727
import nextflow.exception.AbortOperationException
28+
import nextflow.module.ModuleChecksum
2829
import nextflow.module.ModuleSpec
2930
import nextflow.module.ModuleReference
3031
import nextflow.module.ModuleRegistryClient
@@ -60,6 +61,9 @@ class CmdModulePublish extends CmdBase {
6061
@TestOnly
6162
protected ModuleRegistryClient client
6263

64+
//Flag if publish is invoked from a scope/name. In this case we should create/update the .module-info with the correct checksum
65+
private boolean useModuleReference = false
66+
6367
@Override
6468
String getName() {
6569
return 'publish'
@@ -115,14 +119,10 @@ class CmdModulePublish extends CmdBase {
115119

116120
private void publishModule(Path moduleDir, RegistryConfig registryConfig, ModuleSpec manifest){
117121
log.info "Creating module bundle..."
118-
def storage = new ModuleStorage(moduleDir.parent)
119122
def tempBundleFile = Files.createTempFile("nf-module-publish-", ".tar.gz")
120123

121124
try {
122-
storage.createBundle(moduleDir, tempBundleFile)
123-
124-
// Compute bundle checksum
125-
def checksum = storage.computeBundleChecksum(tempBundleFile)
125+
def checksum = ModuleStorage.createBundle(moduleDir, tempBundleFile)
126126
log.info "Bundle checksum: ${checksum}"
127127

128128
// Read bundle content as bytes
@@ -139,6 +139,14 @@ class CmdModulePublish extends CmdBase {
139139
def registryClient = new ModuleRegistryClient(registryConfig)
140140
def response = registryClient.publishModule(manifest.name, request, registryUrl)
141141

142+
if (useModuleReference) {
143+
// If publish is performed using the module reference we should create/update the .module-info with the correct checksum
144+
try {
145+
ModuleChecksum.save(moduleDir, ModuleChecksum.compute(moduleDir))
146+
}catch (Exception e){
147+
log.warn("Unable to save the checksum - ${e.message}")
148+
}
149+
}
142150
println "✓ Module published successfully!"
143151
println ""
144152
println "Module details:"
@@ -246,13 +254,13 @@ class CmdModulePublish extends CmdBase {
246254
return Paths.get(module).toAbsolutePath().normalize()
247255
}
248256

249-
final ref = ModuleReference.parse('@' + module)
257+
final ref = ModuleReference.parse(module)
250258
final localStorage = new ModuleStorage(root ?: Paths.get('.').toAbsolutePath().normalize())
251259

252260
if (!localStorage.isInstalled(ref)){
253261
throw new AbortOperationException("No module diretory found for $module")
254262
}
255-
263+
useModuleReference = true
256264
return localStorage.getModuleDir(ref)
257265
}
258266
}

‎modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRemove.groovy‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,7 @@ class CmdModuleRemove extends CmdBase {
6868
throw new AbortOperationException("Cannot use both -keep-config and -keep-files flags together")
6969
}
7070

71-
def moduleRef = '@' + args[0]
72-
73-
def reference = ModuleReference.parse(moduleRef)
71+
def reference = ModuleReference.parse(args[0])
7472

7573
// Get config
7674
def baseDir = root ?: Paths.get('.').toAbsolutePath().normalize()
@@ -87,15 +85,15 @@ class CmdModuleRemove extends CmdBase {
8785

8886
// Remove local files unless -keep-files is set
8987
if( !keepFiles ) {
90-
println "Removing module files for ${reference.nameWithoutPrefix}..."
88+
println "Removing module files for ${reference}..."
9189
filesRemoved = storage.removeModule(reference)
9290
if( filesRemoved ) {
9391
println "Module files removed successfully"
9492
} else {
95-
println "Module ${reference.nameWithoutPrefix} was not installed locally"
93+
println "Module ${reference} was not installed locally"
9694
}
9795
} else {
98-
println "Keeping module files for ${reference.nameWithoutPrefix} (due to -keep-files flag)"
96+
println "Keeping module files for ${reference} (due to -keep-files flag)"
9997
}
10098

10199
// Remove config entry unless -keep-config is set
@@ -105,17 +103,17 @@ class CmdModuleRemove extends CmdBase {
105103
if( configRemoved ) {
106104
println "Module entry removed from configuration"
107105
} else {
108-
println "Module ${reference.nameWithoutPrefix} was not configured in nextflow_spec.json"
106+
println "Module ${reference} was not configured in nextflow_spec.json"
109107
}
110108
} else {
111109
println "Keeping module entry in nextflow_spec.json (due to -keep-config flag)"
112110
}
113111

114112
// Summary
115113
if( filesRemoved || configRemoved ) {
116-
println "\nModule ${reference.nameWithoutPrefix} removal completed"
114+
println "\nModule ${reference} removal completed"
117115
} else {
118-
println "\nModule ${reference.nameWithoutPrefix} was not found"
116+
println "\nModule ${reference} was not found"
119117
}
120118
}
121119
catch( AbortOperationException e ) {

‎modules/nextflow/src/main/groovy/nextflow/cli/module/CmdModuleRun.groovy‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,12 @@ class CmdModuleRun extends CmdRun {
6161
throw new AbortOperationException("Arguments not provided")
6262
}
6363

64-
// Parse module reference (first argument starting with @)
65-
String moduleRef = '@' + args[0]
66-
6764
// Parse and validate module reference
6865
ModuleReference reference
6966
try {
70-
reference = ModuleReference.parse(moduleRef)
67+
reference = ModuleReference.parse(args[0])
7168
} catch( Exception e ) {
72-
throw new AbortOperationException("Invalid module reference: ${moduleRef}", e)
69+
throw new AbortOperationException("Invalid module reference: ${args[0]}", e)
7370
}
7471

7572
// Get config

‎modules/nextflow/src/main/groovy/nextflow/module/DefaultRemoteModuleResolver.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ class DefaultRemoteModuleResolver implements RemoteModuleResolver {
6767
// Resolve module (will auto-install if missing or version mismatch)
6868
def mainFile = resolver.resolve(reference, null, true)
6969

70-
log.info "Module ${reference.nameWithoutPrefix} resolved to ${mainFile}"
70+
log.debug "Module ${reference} resolved to ${mainFile}"
7171
return mainFile
7272
} catch (Exception e) {
7373
throw new IllegalModulePath(
@@ -97,4 +97,4 @@ class DefaultRemoteModuleResolver implements RemoteModuleResolver {
9797
}
9898
return new ModulesConfig(modules)
9999
}
100-
}
100+
}

‎modules/nextflow/src/main/groovy/nextflow/module/InstalledModule.groovy‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ class InstalledModule {
3939
Path directory
4040
Path mainFile
4141
Path manifestFile
42-
Path checksumFile
42+
Path moduleInfoFile
4343
String installedVersion
4444
String expectedChecksum
4545

@@ -54,9 +54,9 @@ class InstalledModule {
5454
return ModuleIntegrity.CORRUPTED
5555
}
5656

57-
// Check if checksum file exists
58-
if( !Files.exists(checksumFile) ) {
59-
return ModuleIntegrity.MISSING_CHECKSUM
57+
// Check if .module-info file exists
58+
if( !Files.exists(moduleInfoFile) ) {
59+
return ModuleIntegrity.NO_REMOTE_MODULE
6060
}
6161

6262
try {
@@ -71,7 +71,7 @@ class InstalledModule {
7171
return ModuleIntegrity.MODIFIED
7272
}
7373
} catch( Exception e ) {
74-
log.warn "Failed to compute checksum for module ${reference.nameWithoutPrefix}: ${e.message}"
74+
log.warn "Failed to compute checksum for module ${reference}: ${e.message}"
7575
return ModuleIntegrity.CORRUPTED
7676
}
7777
}
@@ -84,6 +84,6 @@ class InstalledModule {
8484
enum ModuleIntegrity {
8585
VALID, // Checksum matches
8686
MODIFIED, // Checksum mismatch (local changes)
87-
MISSING_CHECKSUM, // No .checksum file
87+
NO_REMOTE_MODULE, // No .module-info file (local-only module, no registry origin)
8888
CORRUPTED // Missing required files
8989
}

‎modules/nextflow/src/main/groovy/nextflow/module/ModuleChecksum.groovy‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import java.security.MessageDigest
3333
class ModuleChecksum {
3434

3535
public static final String CHECKSUM_ALGORITHM = "SHA-256"
36-
public static final String CHECKSUM_FILE = ".checksum"
36+
public static final String MODULE_INFO_FILE = ".module-info"
3737

3838
/**
3939
* Compute the SHA-256 checksum of a module directory
@@ -54,7 +54,7 @@ class ModuleChecksum {
5454
try( final walkStream = Files.walk(moduleDir) ) {
5555
walkStream
5656
.filter { Path path -> Files.isRegularFile(path) }
57-
.filter { Path path -> !path.fileName.toString().equals(CHECKSUM_FILE) }
57+
.filter { Path path -> !path.fileName.toString().equals(MODULE_INFO_FILE) }
5858
.sorted()
5959
.each { Path path -> files.add(path) }
6060
}
@@ -85,28 +85,35 @@ class ModuleChecksum {
8585
}
8686

8787
/**
88-
* Save a checksum to the .checksum file in the module directory
88+
* Save a checksum to the .module-info file in the module directory
8989
*
9090
* @param moduleDir The module directory path
9191
* @param checksum The checksum to save
9292
*/
9393
static void save(Path moduleDir, String checksum) {
94-
def checksumFile = moduleDir.resolve(CHECKSUM_FILE)
95-
Files.writeString(checksumFile, checksum)
94+
def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE)
95+
def props = new Properties()
96+
// If file exists loads to update current just checksum property
97+
if( Files.exists( moduleInfoFile))
98+
moduleInfoFile.withInputStream { is -> props.load(is) }
99+
props.setProperty('checksum', checksum)
100+
moduleInfoFile.withOutputStream { os -> props.store(os, null) }
96101
}
97102

98103
/**
99-
* Load a checksum from the .checksum file in the module directory
104+
* Load a checksum from the .module-info file in the module directory
100105
*
101106
* @param moduleDir The module directory path
102107
* @return The checksum, or null if file doesn't exist
103108
*/
104109
static String load(Path moduleDir) {
105-
def checksumFile = moduleDir.resolve(CHECKSUM_FILE)
106-
if( !Files.exists(checksumFile) ) {
110+
def moduleInfoFile = moduleDir.resolve(MODULE_INFO_FILE)
111+
if( !Files.exists(moduleInfoFile) ) {
107112
return null
108113
}
109-
return checksumFile.text
114+
def props = new Properties()
115+
moduleInfoFile.withInputStream { is -> props.load(is) }
116+
return props.getProperty('checksum')
110117
}
111118

112119
/**

‎modules/nextflow/src/main/groovy/nextflow/module/ModuleReference.groovy‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ class ModuleReference {
3434
// Pattern allows: optional @, scope with letters/digits/hyphens/dots/underscores, name segments separated by slashes (no trailing slash)
3535
// Scope: starts with letter/digit, followed by letters/digits/dots/underscores/hyphens
3636
// Name: one or more segments (each starting with letter, followed by letters/digits/underscores/hyphens), separated by slashes
37-
private static final Pattern MODULE_NAME_PATTERN = ~/^@?([a-z0-9][a-z0-9._\-]*)\/([a-z][a-z0-9_\-]*(?:\/[a-z][a-z0-9_\-]*)*)$/
37+
private static final Pattern MODULE_NAME_PATTERN = ~/^([a-z0-9][a-z0-9._\-]*)\/([a-z][a-z0-9._\-]*(?:\/[a-z][a-z0-9._\-]*)*)$/
3838

3939
final String scope
4040
final String name
@@ -43,7 +43,7 @@ class ModuleReference {
4343
ModuleReference(String scope, String name) {
4444
this.scope = scope
4545
this.name = name
46-
this.fullName = "@${scope}/${name}"
46+
this.fullName = "${scope}/${name}"
4747
}
4848

4949
/**
@@ -65,23 +65,14 @@ class ModuleReference {
6565
if( !matcher.matches() ) {
6666
throw new AbortOperationException(
6767
"Invalid module reference: '${source}'. " +
68-
"Expected format: [@]scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " +
68+
"Expected format: scope/name where scope is lowercase alphanumeric with dots/underscores/hyphens " +
6969
"and name is lowercase alphanumeric with underscores/hyphens, optionally with slash-separated segments"
7070
)
7171
}
7272

7373
return new ModuleReference(matcher.group(1), matcher.group(2))
7474
}
7575

76-
/**
77-
* Get the module name without the @ prefix
78-
*
79-
* @return Module name in format "scope/name"
80-
*/
81-
String getNameWithoutPrefix() {
82-
return "${scope}/${name}"
83-
}
84-
8576
@Override
8677
String toString() {
8778
return fullName

0 commit comments

Comments
 (0)