From bf3c93096e1889d6380474983d8f75425fdaf7bb Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 23 Oct 2025 10:08:40 +0200 Subject: [PATCH 1/4] emulate s3-git-remote using platform datalinks Signed-off-by: jorgee --- .../main/groovy/nextflow/cli/CmdRun.groovy | 2 +- .../groovy/nextflow/scm/AssetManager.groovy | 66 +- .../nextflow/scm/RepositoryFactory.groovy | 16 + plugins/nf-tower/build.gradle | 3 +- .../plugin/scm/SeqeraProviderConfig.groovy | 99 +++ .../plugin/scm/SeqeraRepositoryFactory.groovy | 81 +++ .../scm/SeqeraRepositoryProvider.groovy | 225 +++++++ .../plugin/scm/jgit/SeqeraBaseConnection.java | 562 ++++++++++++++++++ .../scm/jgit/SeqeraFetchConnection.java | 132 ++++ .../jgit/SeqeraGitCredentialsProvider.java | 76 +++ .../plugin/scm/jgit/SeqeraPushConnection.java | 201 +++++++ .../plugin/scm/jgit/TransportSeqera.java | 87 +++ 12 files changed, 1546 insertions(+), 4 deletions(-) create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryFactory.groovy create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/TransportSeqera.java diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index bb6d641cc8..7d640b18f4 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -324,6 +324,7 @@ class CmdRun extends CmdBase implements HubOptions { printBanner() // -- resolve main script + Plugins.init() final scriptFile = getScriptFile(pipeline) // -- load command line params @@ -359,7 +360,6 @@ class CmdRun extends CmdBase implements HubOptions { Map configParams = builder.getConfigParams() // -- Load plugins (may register secret providers) - Plugins.init() Plugins.load(config) // -- Initialize real secrets system diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy index b35f1a06d2..047d162f1c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy @@ -112,6 +112,14 @@ class AssetManager { build(pipelineName, config) } + AssetManager(File path, String pipelineName, HubOptions cliOpts = null ) { + assert path + assert pipelineName + // build the object + def config = ProviderConfig.getDefault() + build(path, pipelineName, config, cliOpts) + } + /** * Build the asset manager internal data structure * @@ -135,6 +143,22 @@ class AssetManager { return this } + @PackageScope + AssetManager build( File path, String pipelineName, Map config = null, HubOptions cliOpts = null ) { + + this.providerConfigs = ProviderConfig.createFromMap(config) + + this.project = resolveName(pipelineName) + this.localPath = path + this.hub = checkHubProvider(cliOpts) + this.provider = createHubProvider(hub) + setupCredentials(cliOpts) + validateProjectDir() + + return this + } + + @PackageScope File getLocalGitConfig() { localPath ? new File(localPath,'.git/config') : null @@ -306,8 +330,7 @@ class AssetManager { @PackageScope String resolveNameFromGitUrl( String repository ) { - - final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') + final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') || repository.startsWith('seqera://') if( !isUrl ) return null @@ -685,6 +708,45 @@ class AssetManager { } + /** + * Upload a pipeline to a remote repository + * + * @param revision The revision/branch to upload + * @param remoteName The name of the remote (default: origin) + * @param isNewRepo Whether this is a new repository initialization + * @result A message representing the operation result + */ + String upload(String revision, String remoteName = "origin", boolean isNewRepo = false) { + assert project + assert localPath + + // Create and checkout branch if it doesn't exist + try { + git.checkout().setName(revision).call() + } + catch( Exception ignored ) { + // Branch doesn't exist, create it + git.checkout() + .setCreateBranch(true) + .setName(revision) + .call() + } + + + def pushCommand = git.push() + .setRemote(remoteName) + + pushCommand.add(revision) + + if( provider.hasCredentials() ) + pushCommand.setCredentialsProvider( provider.getGitCredentials() ) + + def result = pushCommand.call() + return "pushed to ${remoteName} (${revision})" + } + + + /** * Clone a pipeline from a remote pipeline repository to the specified folder * diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy index 0856ce24da..19f0b1564c 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/RepositoryFactory.groovy @@ -88,6 +88,7 @@ class RepositoryFactory implements ExtensionPoint { // --== static definitions ==-- private static boolean codeCommitLoaded + private static boolean seqeraLoaded private static List factories0 private static List factories() { @@ -107,6 +108,11 @@ class RepositoryFactory implements ExtensionPoint { codeCommitLoaded=true factories0=null } + if( config.name=='seqera' || config.platform == 'seqera' && !seqeraLoaded){ + Plugins.startIfMissing('nf-tower') + seqeraLoaded=true + factories0=null + } // scan all installed Git repository factories and find the first // returning an provider instance for the specified parameters @@ -125,6 +131,11 @@ class RepositoryFactory implements ExtensionPoint { codeCommitLoaded=true factories0=null } + if( (name=='seqera' || attrs.platform=='seqera') && !seqeraLoaded ) { + Plugins.startIfMissing('nf-tower') + seqeraLoaded=true + factories0=null + } final config = factories().findResult( it -> it.createConfigInstance(name, attrs) ) if( !config ) { @@ -139,6 +150,11 @@ class RepositoryFactory implements ExtensionPoint { codeCommitLoaded=true factories0=null } + if( url.protocol.equals('seqera') && !seqeraLoaded){ + Plugins.startIfMissing('nf-tower') + seqeraLoaded=true + factories0=null + } final provider = factories().findResult( it -> it.getConfig(providers, url) ) if( !provider ) { diff --git a/plugins/nf-tower/build.gradle b/plugins/nf-tower/build.gradle index 484374f212..460097f92f 100644 --- a/plugins/nf-tower/build.gradle +++ b/plugins/nf-tower/build.gradle @@ -26,7 +26,8 @@ nextflowPlugin { 'io.seqera.tower.plugin.TowerFactory', 'io.seqera.tower.plugin.TowerFusionToken', 'io.seqera.tower.plugin.auth.AuthCommandImpl', - 'io.seqera.tower.plugin.launch.LaunchCommandImpl' + 'io.seqera.tower.plugin.launch.LaunchCommandImpl', + 'io.seqera.tower.plugin.scm.SeqeraRepositoryFactory' ] } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy new file mode 100644 index 0000000000..5678f1246a --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy @@ -0,0 +1,99 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.tower.plugin.BaseCommandImpl +import nextflow.Const +import nextflow.Global +import nextflow.SysEnv +import nextflow.config.ConfigBuilder +import nextflow.exception.AbortOperationException +import nextflow.platform.PlatformHelper +import nextflow.scm.ProviderConfig + +/** + * Implements a provider config for Seqera Platform data-links git-remote repositories + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class SeqeraProviderConfig extends ProviderConfig { + + private String endpoint = 'https://api.cloud.seqera.io' + private String accessToken + private String workspaceId + + SeqeraProviderConfig(String name, Map values) { + super(name, [server: "seqera://$name"] + values) + setValues(values) + } + + SeqeraProviderConfig(String name) { + super(name, [platform: 'seqera', server: "seqera://$name"]) + setValues() + } + + private void setValues(Map values = Map.of()) { + // Get tower config from session if exists + def towerConfig = getDefaultValues() + + + // Merge with SCM values + final config = towerConfig + values + endpoint = PlatformHelper.getEndpoint(config, SysEnv.get()) + accessToken = PlatformHelper.getAccessToken(config, SysEnv.get()) + workspaceId = PlatformHelper.getWorkspaceId(config, SysEnv.get()) + + if (!accessToken) { + throw new AbortOperationException("Seqera Platform access token not configured. Set TOWER_ACCESS_TOKEN environment variable or configure it in nextflow.config") + } + } + + String getEndpoint() { + this.endpoint + } + + String getAccessToken() { + this.accessToken + } + + String getWorkspaceId() { + this.workspaceId + } + + @Override + protected String resolveProjectName(String path) { + log.debug("Resolving project name from $path") + if (!server.startsWith('seqera://')) { + throw new AbortOperationException("Seqera project server doesn't start with seqera://") + } + return "${server.substring('seqera://'.size())}/$path" + } + + /** + * Get the default values from the session or the global config + */ + private Map getDefaultValues() { + Map defaultValues = Global.session?.config?.tower as Map + if(!defaultValues) + defaultValues = new ConfigBuilder().setHomeDir(Const.APP_HOME_DIR).setCurrentDir(Const.APP_HOME_DIR).buildConfigObject()?.tower as Map + return defaultValues ?: Map.of() + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryFactory.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryFactory.groovy new file mode 100644 index 0000000000..e6ece14a71 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryFactory.groovy @@ -0,0 +1,81 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.tower.plugin.scm.jgit.TransportSeqera +import nextflow.plugin.Priority +import nextflow.scm.GitUrl +import nextflow.scm.ProviderConfig +import nextflow.scm.RepositoryFactory +import nextflow.scm.RepositoryProvider + +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Implements a factory to create an instance of {@link SeqeraRepositoryProvider} + * + * @author Jorge Ejarque + */ +@Slf4j +@Priority(-10) +@CompileStatic +class SeqeraRepositoryFactory extends RepositoryFactory { + + private static AtomicBoolean registered = new AtomicBoolean(false) + + @Override + protected RepositoryProvider createProviderInstance(ProviderConfig config, String project) { + if (!registered.get()) { + registered.set(true) + TransportSeqera.register() + } + + return config.platform == 'seqera' + ? new SeqeraRepositoryProvider(project, config) + : null + } + + @Override + protected ProviderConfig getConfig(List providers, GitUrl url) { + // Only handle seqera:// URLs + if (url.protocol != 'seqera') { + return null + } + + // Seqera repository config depends on the data-link ID stored as domain + def config = providers.find { it -> it.domain == url.domain } + if (config) { + log.debug "Git url=$url (1) -> config=$config" + return config + } + + // Create a new instance if not found + config = new SeqeraProviderConfig(url.domain) + + return config + } + + @Override + protected ProviderConfig createConfigInstance(String name, Map attrs) { + final copy = new HashMap(attrs) + return copy.platform == 'seqera' + ? new SeqeraProviderConfig(name, copy) + : null + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy new file mode 100644 index 0000000000..6d40bb05d5 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy @@ -0,0 +1,225 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm + +import groovy.transform.CompileStatic +import groovy.transform.Memoized +import groovy.util.logging.Slf4j +import io.seqera.tower.plugin.scm.jgit.SeqeraGitCredentialsProvider +import nextflow.exception.AbortOperationException +import nextflow.scm.ProviderConfig +import nextflow.scm.RepositoryProvider +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.api.errors.TransportException +import org.eclipse.jgit.lib.Constants +import org.eclipse.jgit.lib.Ref +import org.eclipse.jgit.transport.CredentialsProvider + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Implements a repository provider for Seqera Platform data-links git-remote repositories. + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class SeqeraRepositoryProvider extends RepositoryProvider { + + SeqeraRepositoryProvider(String project, ProviderConfig config) { + assert config instanceof SeqeraProviderConfig + log.debug("Creating Seqera repository provider for $project") + this.project = project + this.config = config + } + + /** {@inheritDoc} **/ + @Memoized + @Override + CredentialsProvider getGitCredentials() { + final providerConfig = this.config as SeqeraProviderConfig + final credentials = new SeqeraGitCredentialsProvider() + if (providerConfig.endpoint) { + credentials.setEndpoint(providerConfig.endpoint) + } + if (providerConfig.accessToken) { + credentials.setAccessToken(providerConfig.accessToken) + } + if (providerConfig.workspaceId) { + credentials.setWorkspaceId(providerConfig.workspaceId) + } + return credentials + } + + /** {@inheritDoc} **/ + // called by AssetManager + // used to set credentials for a clone, pull, fetch, operation + @Override + boolean hasCredentials() { + // set to true + // uses Seqera Platform credentials instead of username : password + // see getGitCredentials() + return true + } + + /** {@inheritDoc} **/ + @Override + String getName() { return project } + + /** {@inheritDoc} **/ + @Override + String getEndpointUrl() { + return "seqera://$project" + } + + /** {@inheritDoc} **/ + // not used, but the abstract method needs to be overridden + @Override + String getContentUrl(String path) { + throw new UnsupportedOperationException() + } + + /** {@inheritDoc} **/ + // called by AssetManager + @Override + String getCloneUrl() { getEndpointUrl() } + + /** {@inheritDoc} **/ + // called by AssetManager + @Override + String getRepositoryUrl() { getEndpointUrl() } + + /** + * {@inheritDoc} + * + * Note: Seqera git-remote stores repositories as Git bundles in data-links (one bundle per branch). + * Reading a single file requires downloading and unpacking the entire bundle for that branch. + * When no revision is specified, we determine the default branch from the remote HEAD + * to avoid downloading unnecessary branches. + */ + @Override + byte[] readBytes(String path) { + log.debug("Reading $path from Seqera git-remote") + Path tmpDir = null + try { + tmpDir = Files.createTempDirectory("seqera-git-remote-") + + // Determine which branch to clone + def branchToClone = revision + if (!branchToClone) { + // No revision specified - fetch only the default branch + // This avoids downloading unnecessary branch bundles + branchToClone = getDefaultBranch() + log.debug("No revision specified, using default branch: $branchToClone") + } + + final command = Git.cloneRepository() + .setURI(getEndpointUrl()) + .setDirectory(tmpDir.toFile()) + .setCredentialsProvider(getGitCredentials()) + .setCloneAllBranches(false) // Only clone the specified branch + .setBranch(branchToClone) + + command.call() + final file = tmpDir.resolve(path) + return Files.exists(file) ? Files.readAllBytes(file) : null + } + catch (Exception e) { + log.debug("Unable to retrieve file: $path from repo: $project", e) + return null + } + finally { + if (tmpDir != null && Files.exists(tmpDir)) { + tmpDir.toFile().deleteDir() + } + } + } + + /** + * Get the default branch from the Seqera git-remote repository by querying remote refs. + * Uses Git's lsRemote to fetch the HEAD symbolic ref, which points to the default branch. + * + * @return The default branch name + */ + @Memoized + String getDefaultBranch() { + // Fetch remote refs using Git's lsRemote + final refs = fetchRefs() + if (!refs) { + throw new Exception("No remote references found") + } + // Find the HEAD symbolic ref + final headRef = refs.find { it.name == Constants.HEAD } + + if (!headRef) { + throw new Exception("No remote HEAD ref found") + } + + if (!headRef.isSymbolic()) { + throw new Exception("Incorrect HEAD ref. Not a symbolic ref.") + } + + final target = headRef.target.name + if (target.startsWith('refs/heads/')) { + return target.substring('refs/heads/'.length()) + } + return target + } + + @Override + List listDirectory(String path, int depth) { + throw new UnsupportedOperationException("Seqera-git-remote does not support 'listDirectory' operation") + } + + /** {@inheritDoc} **/ + // called by AssetManager + @Override + void validateRepo() { + // Nothing to check + } + + private String errMsg(Exception e) { + def msg = "Unable to access Git repository" + if (e.message) { + msg + " - ${e.message}" + } else { + msg += ": " + getCloneUrl() + } + return msg + } + + @Override + List getBranches() { + try { + return super.getBranches() + } + catch (TransportException e) { + throw new AbortOperationException(errMsg(e), e) + } + } + + @Override + List getTags() { + try { + return super.getTags() + } + catch (TransportException e) { + throw new AbortOperationException(errMsg(e), e) + } + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java new file mode 100644 index 0000000000..1db342babc --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java @@ -0,0 +1,562 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm.jgit; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import io.seqera.http.HxClient; +import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.transport.Connection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Base class for connections with Seqera Platform data-links git-remote compatibility + * + * @author Jorge Ejarque + * + */ +public class SeqeraBaseConnection implements Connection { + + protected static final Logger log = LoggerFactory.getLogger(SeqeraBaseConnection.class); + protected TransportSeqera transport; + protected DataLink dataLink; + protected String repoPath; + protected HxClient httpClient; + protected String endpoint; + protected String workspaceId; + private final Map advertisedRefs = new HashMap(); + private final Gson gson = new Gson(); + + + public SeqeraBaseConnection(TransportSeqera transport) { + this.transport = transport; + // Parse URI: seqera://bucket/path + String dataLinkName = transport.getURI().getHost(); + this.repoPath = transport.getURI().getPath().substring(1); // Remove leading slash + SeqeraGitCredentialsProvider credentials = (SeqeraGitCredentialsProvider) transport.getCredentialsProvider(); + this.endpoint = credentials.getEndpoint(); + this.workspaceId = credentials.getWorkspaceId(); + this.httpClient = createHttpClient(credentials); + this.dataLink = findDataLink(dataLinkName); + loadRefsMap(); + log.debug("Created Seqera Connection for dataLink={} path={}", dataLink.id, repoPath); + } + + protected HxClient createHttpClient(SeqeraGitCredentialsProvider credentials) { + final HxClient.Builder builder = HxClient.newBuilder(); + + // Set up authentication + final String token = credentials.getAccessToken(); + // Count occurrences of '.' in token + long dotCount = token.chars().filter(ch -> ch == '.').count(); + if (dotCount == 2) { + builder.bearerToken(token); + } else { + try { + final String plain = new String(java.util.Base64.getDecoder().decode(token)); + final int p = plain.indexOf('.'); + if (p != -1) { + builder.bearerToken(token); + } else { + builder.basicAuth("@token:" + token); + } + } catch (Exception e) { + builder.basicAuth("@token:" + token); + } + } + + return builder + .followRedirects(HttpClient.Redirect.NORMAL) + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(java.time.Duration.ofSeconds(60)) + .build(); + } + + + protected String getDefaultBranchRef() throws IOException { + String path = repoPath + "/HEAD"; + byte[] content = downloadFile(path); + if (content != null) { + return new String(content); + } + return null; + } + + /** + * Find DatalinkId from the bucket name + */ + protected DataLink findDataLink(String linkName) { + try { + String url = endpoint + "/data-links?status=AVAILABLE&search=" + linkName; + if (workspaceId != null) { + url += "&workspaceId=" + workspaceId; + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() < 200 && response.statusCode() >= 300) { + String message = response.body(); + if (message == null) message = ""; + message = message + " - HTTP " + response.statusCode(); + throw new RuntimeException("Failed to find data-link: " + linkName + message); + } + + JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); + JsonArray files = responseJson.getAsJsonArray("dataLinks"); + + List dataLinks = java.util.stream.StreamSupport.stream(files.spliterator(), false) + .map(JsonElement::getAsJsonObject) + .filter(obj -> obj.get("name").getAsString().equals(linkName)) + .map(obj -> getDataLinkFromJSONObject(obj)) + .toList(); + + if (dataLinks.isEmpty()) { + log.debug("No datalink response: " + linkName); + throw new RuntimeException("No Data-link found for " + linkName); + } + return dataLinks.getFirst(); + } catch (IOException e) { + throw new RuntimeException("Exception finding data-link", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Download interrupted", e); + } + } + + private DataLink getDataLinkFromJSONObject(JsonObject obj) { + String id = obj.get("id").getAsString(); + //Get first credential + String credentialsId = obj.get("credentials").getAsJsonArray().get(0).getAsJsonObject().get("id").getAsString(); + + return new DataLink(id, credentialsId); + } + + /** + * Download a file from the data-link + */ + protected byte[] downloadFile(String filePath) throws IOException { + try { + Map queryParams = new HashMap<>(); + if (workspaceId != null) { + queryParams.put("workspaceId", workspaceId); + } + if (dataLink.credentialsId != null) { + queryParams.put("credentialsId", dataLink.credentialsId); + } + + String url = buildDataLinkUrl("/download/"+ URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); + + if (response.statusCode() == 404) { + log.debug("File {} in data-link {} not found", filePath, dataLink.id); + return null; + } + + if (response.statusCode() >= 400) { + throw new IOException("Failed to download file: " + filePath + " - status: " + response.statusCode()); + } + + return response.body(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Download interrupted", e); + } + } + + /** + * Download a file from the data-link directly to a local file + * This is more memory-efficient for large files like bundles + */ + protected void downloadFileToPath(String filePath, Path targetPath) throws IOException { + try { + Map queryParams = new HashMap<>(); + if (workspaceId != null) { + queryParams.put("workspaceId", workspaceId); + } + if (dataLink.credentialsId != null) { + queryParams.put("credentialsId", dataLink.credentialsId); + } + + String url = buildDataLinkUrl("/download/"+ URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofFile(targetPath)); + + if (response.statusCode() == 404) { + log.debug("File {} in data-link {} not found", filePath, dataLink.id); + throw new IOException("File not found: " + filePath); + } + + if (response.statusCode() >= 400) { + throw new IOException("Failed to download file: " + filePath + " - status: " + response.statusCode()); + } + + log.debug("Downloaded {} to {}", filePath, targetPath); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Download interrupted", e); + } + } + + /** + * Upload a file to the data-link + */ + protected void uploadFile(String filePath, Path localFile) throws IOException { + try { + // Step 1: Get upload URL + Map queryParams = new HashMap<>(); + if (workspaceId != null) { + queryParams.put("workspaceId", workspaceId); + } + if (dataLink.credentialsId != null) { + queryParams.put("credentialsId", dataLink.credentialsId); + } + String url = buildDataLinkUrl("/upload", queryParams); + + JsonObject uploadRequest = new JsonObject(); + uploadRequest.addProperty("fileName", filePath); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(gson.toJson(uploadRequest))) + .build(); + + HttpResponse response = httpClient.sendAsString(request); + + if (response.statusCode() >= 400) { + throw new IOException("Failed to get upload URL for: " + filePath + " - status: " + response.statusCode()); + } + + JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); + String uploadUrl = responseJson.get("uploadUrl").getAsString(); + + // Step 2: Upload file to the provided URL + HttpRequest uploadFileRequest = HttpRequest.newBuilder() + .uri(URI.create(uploadUrl)) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) + .build(); + + HttpResponse uploadResponse = httpClient.send(uploadFileRequest, HttpResponse.BodyHandlers.discarding()); + + if (uploadResponse.statusCode() >= 400) { + throw new IOException("Failed to upload file: " + filePath + " - status: " + uploadResponse.statusCode()); + } + + // Step 3: Finish upload (required for some providers like AWS) + + String finishUrl = buildDataLinkUrl("/upload/finish", Map.of()); + if (workspaceId != null) { + finishUrl += "?workspaceId=" + workspaceId; + } + + JsonObject finishRequest = new JsonObject(); + finishRequest.addProperty("fileName", filePath); + + HttpRequest finishReq = HttpRequest.newBuilder() + .uri(URI.create(finishUrl)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(gson.toJson(finishRequest))) + .build(); + + httpClient.send(finishReq, HttpResponse.BodyHandlers.discarding()); + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Upload interrupted", e); + } + } + + /** + * List files in a path + */ + protected List listFiles(String path) throws IOException { + Map queryParams = new HashMap<>(); + if (workspaceId != null) { + queryParams.put("workspaceId", workspaceId); + } + if (dataLink.credentialsId != null) { + queryParams.put("credentialsId", dataLink.credentialsId); + } + String url = buildDataLinkUrl("/browse/" + URLEncoder.encode(path, StandardCharsets.UTF_8), queryParams); + log.debug(" GET {}", url); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build(); + + HttpResponse response = httpClient.sendAsString(request); + + if (response.statusCode() == 404) { + log.debug("No files found for {}", url); + return List.of(); + } + + if (response.statusCode() >= 400) { + log.debug("Error getting files {}", url); + String message = response.body(); + if (message == null) message = ""; + message = message + " - HTTP " + response.statusCode(); + + throw new IOException("Failed to list files in: " + path + message); + } + log.debug(response.body()); + JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); + JsonArray files = responseJson.getAsJsonArray("objects"); + + return StreamSupport.stream(files.spliterator(), false) + .map(JsonElement::getAsJsonObject) + .map(obj -> obj.get("name").getAsString()) + .collect(java.util.stream.Collectors.toList()); + } + + /** + * Delete a file from the data-link + */ + protected void deleteFile(String filePath) throws IOException { + try { + Map queryParams = new HashMap<>(); + + if (workspaceId != null) { + queryParams.put("workspaceId", workspaceId); + } + String url = buildDataLinkUrl("/content", queryParams); + + JsonObject deleteRequest = new JsonObject(); + deleteRequest.addProperty("path", filePath); + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Content-Type", "application/json") + .method("DELETE", HttpRequest.BodyPublishers.ofString(gson.toJson(deleteRequest))) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); + + if (response.statusCode() >= 400 && response.statusCode() != 404) { + throw new IOException("Failed to delete file: " + filePath + " - status: " + response.statusCode()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Delete interrupted", e); + } + } + + private String buildDataLinkUrl(String path, Map queryParams) { + final StringBuilder url = new StringBuilder(endpoint + "/data-links/" + URLEncoder.encode(dataLink.id, StandardCharsets.UTF_8)); + if (path != null) { + if (!path.startsWith("/")) url.append("/"); + url.append(path); + } + + if (queryParams != null && !queryParams.isEmpty()) { + String queryString = queryParams.entrySet().stream() + .map(entry -> URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8) + + "=" + + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)) + .collect(Collectors.joining("&")); + url.append('?').append(queryString); + } + return url.toString(); + } + + private void loadRefsMap() { + log.debug("Loading refs Maps"); + try { + addRefs("heads"); + } catch (Exception e) { + log.debug("No heads found for dataLink={} path={}: {}", dataLink.id, repoPath, e.getMessage()); + } + try { + addRefs("tags"); + } catch (Exception e) { + log.debug("No tags found for dataLink={} path={}: {}", dataLink.id, repoPath, e.getMessage()); + } + try { + final String defaultBranch = getDefaultBranchRef(); + if (defaultBranch != null) { + Ref target = advertisedRefs.get(defaultBranch); + if (target != null) { + advertisedRefs.put(Constants.HEAD, new SymbolicRef(Constants.HEAD, target)); + } + } + } catch (Exception e) { + log.debug("No default refs found for dataLink={} path={}: {}", dataLink.id, repoPath, e.getMessage()); + } + } + + private void addRefs(String refType) throws IOException { + String path = repoPath + "/refs/" + refType; + List branches = listFiles(path); + + if (branches == null || branches.isEmpty()) { + log.debug("No {} refs found for dataLink={} path={}", refType, dataLink.id, repoPath); + return; + } + + for (String branch : branches) { + String branchPath = path + "/" + branch; + List bundles = listFiles(branchPath); + if (bundles != null && !bundles.isEmpty()) { + // Get the first bundle (there should only be one per branch) + String bundleName = bundles.get(0); + addRef(refType, branch, bundleName); + } + } + } + + private void addRef(String refType, String branchName, String bundleName) { + String sha = bundleName.replace(".bundle", ""); + ObjectId objectId = ObjectId.fromString(sha); + String refName = "refs/" + refType + "/" + branchName; + if (refName.endsWith("/")) + refName = refName.substring(0, refName.length() - 1); + if ("heads".equals(refType)) { + advertisedRefs.put(refName, new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, refName, objectId)); + } else if ("tags".equals(refType)) { + advertisedRefs.put(refName, new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, refName, objectId)); + } + } + + @Override + public Map getRefsMap() { + return advertisedRefs; + } + + @Override + public Collection getRefs() { + return advertisedRefs.values(); + } + + @Override + public Ref getRef(String name) { + return advertisedRefs.get(name); + } + + @Override + public void close() { + // Cleanup if needed + } + + @Override + public String getMessages() { + return ""; + } + + @Override + public String getPeerUserAgent() { + return ""; + } + + static class BranchData { + private String type; + private String simpleName; + private String refName; + private ObjectId objectId; + + private BranchData(String type, String simpleName, ObjectId objectId) { + this.type = type; + this.simpleName = simpleName; + this.refName = String.format("refs/%s/%s", type, simpleName); + this.objectId = objectId; + } + + public static BranchData fromKey(String key) { + String[] parts = key.split("/"); + if (parts.length < 5) { + throw new RuntimeException("Incorrect key format in Seqera git-remote repository. Key should include: repo-path/refs///.bundle"); + } + final String type = parts[parts.length - 3]; + final String branch = parts[parts.length - 2]; + final String sha = parts[parts.length - 1].replace(".bundle", ""); + return new BranchData(type, branch, ObjectId.fromString(sha)); + } + + public String getType() { + return type; + } + + public String getSimpleName() { + return simpleName; + } + + public String getRefName() { + return refName; + } + + public ObjectId getObjectId() { + return objectId; + } + } + + static class DataLink { + private String id; + private String credentialsId; + + private DataLink(String id, String credentialsId) { + this.id = id; + this.credentialsId = credentialsId; + } + + public String getId() { + return id; + } + + public String getCredentialsId() { + return credentialsId; + } + + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java new file mode 100644 index 0000000000..104038596e --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java @@ -0,0 +1,132 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm.jgit; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.transport.*; +import org.eclipse.jgit.util.FileUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +/** + * Fetch Connection implementation for Seqera Platform data-links git-remote storage. + * + * @author Jorge Ejarque + */ +public class SeqeraFetchConnection extends SeqeraBaseConnection implements FetchConnection { + + public SeqeraFetchConnection(TransportSeqera transport) { + super(transport); + } + + @Override + public void fetch(ProgressMonitor monitor, Collection want, Set have) throws TransportException { + Path tmpdir = null; + try { + tmpdir = Files.createTempDirectory("seqera-remote-git-"); + for (Ref r : want) { + downloadBundle(r, have, tmpdir, monitor); + } + } catch (IOException e) { + throw new TransportException(transport.getURI(), "Exception fetching branches", e); + } finally { + if (tmpdir != null) { + try { + FileUtils.delete(tmpdir.toFile(), FileUtils.RECURSIVE); + } catch (IOException e) { + throw new TransportException(transport.getURI(), "Exception cleaning up temporary files", e); + } + } + } + } + + private void downloadBundle(Ref r, Set have, Path tmpdir, ProgressMonitor monitor) throws IOException { + log.debug("Fetching {} in {}", r.getName(), tmpdir); + + // List bundles in the ref directory + String refPath = repoPath + "/" + r.getName(); + List bundles = listFiles(refPath); + + if (bundles == null || bundles.isEmpty()) { + throw new TransportException(transport.getURI(), "No bundle for " + r.getName()); + } + + if (bundles.size() > 1) { + throw new TransportException(transport.getURI(), "More than one bundle for " + r.getName()); + } + + String bundleName = bundles.get(0); + String bundlePath = refPath + "/" + bundleName; + + Path localBundle = tmpdir.resolve(bundleName); + Files.createDirectories(localBundle.getParent()); + log.trace("Downloading bundle {} for branch {} to {}", bundlePath, r.getName(), localBundle); + + // Download the bundle + downloadFileToPath(bundlePath, localBundle); + + parseBundle(r, have, localBundle, monitor); + } + + private void parseBundle(Ref r, Set have, Path localBundle, ProgressMonitor monitor) throws TransportException { + List specs = new ArrayList<>(); + List refs = new ArrayList<>(); + refs.add(r); + specs.add(new RefSpec().setForceUpdate(true).setSourceDestination(Constants.R_REFS + '*', Constants.R_REFS + '*')); + + try (FetchConnection c = Transport.open(transport.getLocal(), new URIish(localBundle.toUri().toString())).openFetch(specs)) { + c.fetch(monitor, refs, have); + } catch (IOException | RuntimeException | URISyntaxException err) { + close(); + throw new TransportException(transport.getURI(), err.getMessage(), err); + } + } + + @Override + public void fetch(ProgressMonitor monitor, Collection want, Set have, OutputStream out) throws TransportException { + fetch(monitor, want, have); + } + + @Override + public boolean didFetchIncludeTags() { + return false; + } + + @Override + public boolean didFetchTestConnectivity() { + return false; + } + + @Override + public void setPackLockMessage(String message) { + // No pack lock message supported. + } + + @Override + public Collection getPackLocks() { + return Collections.emptyList(); + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java new file mode 100644 index 0000000000..2693d64945 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java @@ -0,0 +1,76 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm.jgit; + +import org.eclipse.jgit.errors.UnsupportedCredentialItem; +import org.eclipse.jgit.transport.CredentialItem; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.URIish; + +/** + * JGit credentials provider wrapper for the Seqera Platform API credentials and configuration. + * + * @author Jorge Ejarque + */ +public class SeqeraGitCredentialsProvider extends CredentialsProvider { + + private String endpoint; + private String accessToken; + private String workspaceId; + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public void setAccessToken(String accessToken) { + this.accessToken = accessToken; + } + + public void setWorkspaceId(String workspaceId) { + this.workspaceId = workspaceId; + } + + public String getEndpoint() { + return endpoint != null ? endpoint : "https://api.cloud.seqera.io"; + } + + public String getAccessToken() { + if (accessToken == null) { + throw new IllegalStateException("Seqera Platform access token not configured"); + } + return accessToken; + } + + public String getWorkspaceId() { + return workspaceId; + } + + @Override + public boolean isInteractive() { + return false; + } + + @Override + public boolean supports(CredentialItem... items) { + return false; + } + + @Override + public boolean get(URIish uri, CredentialItem... items) throws UnsupportedCredentialItem { + return false; + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java new file mode 100644 index 0000000000..f53711f47a --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java @@ -0,0 +1,201 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm.jgit; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.transport.BundleWriter; +import org.eclipse.jgit.transport.PushConnection; +import org.eclipse.jgit.transport.RemoteRefUpdate; +import org.eclipse.jgit.util.FileUtils; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Push connection implementation for Seqera Platform data-links git-remote storage. + * + * @author Jorge Ejarque + */ +public class SeqeraPushConnection extends SeqeraBaseConnection implements PushConnection { + + public SeqeraPushConnection(TransportSeqera transport) { + super(transport); + } + + @Override + public void push(ProgressMonitor monitor, Map refUpdates) throws TransportException { + Path tmpdir = null; + try { + tmpdir = Files.createTempDirectory("seqera-remote-git-"); + for (Map.Entry entry : refUpdates.entrySet()) { + pushBranch(entry, tmpdir); + } + } catch (IOException e) { + throw new TransportException(transport.getURI(), "Exception pushing branches", e); + } finally { + if (tmpdir != null) { + try { + FileUtils.delete(tmpdir.toFile(), FileUtils.RECURSIVE); + } catch (IOException e) { + throw new TransportException(transport.getURI(), "Exception cleaning up temporary files", e); + } + } + } + } + + private void pushBranch(Map.Entry entry, Path tmpdir) throws IOException { + log.debug("Pushing {} reference", entry.getKey()); + final Ref ref = transport.getLocal().findRef(entry.getKey()); + if (ref == null || ref.getObjectId() == null) { + throw new IllegalStateException("Branch " + entry.getKey() + " not found"); + } + + String oldBundlePath = checkExistingBundle(entry.getKey()); + if (oldBundlePath != null && isSameObjectId(oldBundlePath, ref.getObjectId())) { + setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.UP_TO_DATE); + return; + } + + if (oldBundlePath != null && !isCommitInBranch(oldBundlePath, ref)) { + setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.REJECTED_REMOTE_CHANGED); + return; + } + + log.trace("Generating bundle for branch {} in {}", entry.getKey(), tmpdir); + Path bundleFile = bundle(ref, tmpdir); + String bundlePath = repoPath + "/" + entry.getKey() + "/" + bundleFile.getFileName().toString(); + + log.trace("Uploading bundle {} to data-link {}", bundleFile, dataLink.getId()); + uploadFile(bundlePath, bundleFile); + + if (oldBundlePath != null) { + log.trace("Deleting old bundle {}", oldBundlePath); + deleteFile(oldBundlePath); + } + + setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.OK); + + if (getRef(Constants.HEAD) == null) { + updateRemoteHead(entry.getKey()); + } + } + + private void updateRemoteHead(String ref) { + try { + String headPath = repoPath + "/HEAD"; + // Try to download HEAD to check if it exists + byte[] existingHead = downloadFile(headPath); + if (existingHead == null) { + log.debug("No remote default branch. Setting to {}.", ref); + // Create a temporary file with the ref content + Path tempFile = Files.createTempFile("head-", ".txt"); + try { + Files.write(tempFile, ref.getBytes(StandardCharsets.UTF_8)); + uploadFile(headPath, tempFile); + } finally { + Files.deleteIfExists(tempFile); + } + } + } catch (IOException e) { + log.warn("Failed to update remote HEAD", e); + } + } + + private boolean isSameObjectId(String bundlePath, ObjectId commitId) { + BranchData branch = BranchData.fromKey(bundlePath); + return branch.getObjectId().name().equals(commitId.name()); + } + + /** + * Sets the status on a RemoteRefUpdate using reflection. + * This is necessary because RemoteRefUpdate.status is package-private and JGit + * doesn't provide a public API to set it. It also JAR signing verification which + * disables the implementations of this class in the org.eclipse.jgit.transport package. + * The Custom transport implementations like this Seqera transport need to update the + * RemoteRefUpdate.status to inform callers of push results. + */ + private void setUpdateStatus(RemoteRefUpdate update, RemoteRefUpdate.Status status) { + try { + Field statusField = RemoteRefUpdate.class.getDeclaredField("status"); + statusField.setAccessible(true); + statusField.set(update, status); + } catch (NoSuchFieldException e) { + throw new RuntimeException("JGit API changed: RemoteRefUpdate.status field not found. " + + "This may require updating the transport implementation.", e); + } catch (IllegalAccessException e) { + throw new RuntimeException("Unable to access RemoteRefUpdate.status field", e); + } catch (Exception e) { + throw new RuntimeException("Unexpected error setting status on RemoteRefUpdate", e); + } + } + + public boolean isCommitInBranch(String bundlePath, Ref branchRef) throws IOException { + ObjectId commitId = BranchData.fromKey(bundlePath).getObjectId(); + try (RevWalk walk = new RevWalk(transport.getLocal())) { + RevCommit branchTip = walk.parseCommit(branchRef.getObjectId()); + RevCommit targetCommit = walk.parseCommit(commitId); + + // Check if the commit is reachable from the branch tip + return walk.isMergedInto(targetCommit, branchTip); + } + } + + private String checkExistingBundle(String refName) throws IOException { + String refPath = repoPath + "/" + refName; + List bundles = listFiles(refPath); + + if (bundles == null || bundles.isEmpty()) { + return null; + } + + if (bundles.size() > 1) { + throw new TransportException(transport.getURI(), "More than one bundle for " + refName); + } + + return refPath + "/" + bundles.get(0); + } + + @Override + public void push(ProgressMonitor monitor, Map refUpdates, OutputStream out) throws TransportException { + push(monitor, refUpdates); + } + + private Path bundle(Ref ref, Path tmpdir) throws IOException { + final BundleWriter writer = new BundleWriter(transport.getLocal()); + Path bundleFile = tmpdir.resolve(ref.getObjectId().name() + ".bundle"); + writer.include(ref); + try (OutputStream out = new FileOutputStream(bundleFile.toFile())) { + writer.writeBundle(NullProgressMonitor.INSTANCE, out); + } + return bundleFile; + } + + @Override + public void close() { + // Cleanup if needed + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/TransportSeqera.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/TransportSeqera.java new file mode 100644 index 0000000000..3aa756fff4 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/TransportSeqera.java @@ -0,0 +1,87 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.scm.jgit; + +import org.eclipse.jgit.errors.TransportException; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.*; + +import java.util.Collections; +import java.util.Set; + +/** + * JGit transport implementation for Seqera Platform data-links git-remote storage. + * + * @author Jorge Ejarque + */ +public class TransportSeqera extends Transport { + + public static final TransportProtocol PROTO_SEQERA = new SeqeraTransportProtocol(); + + public TransportSeqera(Repository local, URIish uri) throws TransportException { + super(local, uri); + } + + @Override + public FetchConnection openFetch() throws TransportException { + return new SeqeraFetchConnection(this); + } + + @Override + public PushConnection openPush() throws TransportException { + return new SeqeraPushConnection(this); + } + + @Override + public void close() { + // cleanup resources if needed + } + + public Repository getLocal() { + return this.local; + } + + public static class SeqeraTransportProtocol extends TransportProtocol { + @Override + public String getName() { + return "Seqera Platform"; + } + + @Override + public Set getSchemes() { + return Collections.singleton("seqera"); + } + + @Override + public boolean canHandle(URIish uri, Repository local, String remoteName) { + return "seqera".equals(uri.getScheme()); + } + + @Override + public Transport open(URIish uri, Repository local, String remoteName) throws TransportException { + try { + return new TransportSeqera(local, uri); + } catch (TransportException e) { + throw e; + } + } + } + + public static void register() { + Transport.register(PROTO_SEQERA); + } +} From 6a10e6a448ef4ace3e18c711bb0e0b491664451c Mon Sep 17 00:00:00 2001 From: jorgee Date: Tue, 28 Oct 2025 15:48:45 +0100 Subject: [PATCH 2/4] Updating base connection Signed-off-by: jorgee --- .../plugin/scm/jgit/SeqeraBaseConnection.java | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java index 1db342babc..40a21be6e3 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java @@ -262,11 +262,12 @@ protected void uploadFile(String filePath, Path localFile) throws IOException { if (dataLink.credentialsId != null) { queryParams.put("credentialsId", dataLink.credentialsId); } - String url = buildDataLinkUrl("/upload", queryParams); + String url = buildDataLinkUrl("/upload/" + URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); JsonObject uploadRequest = new JsonObject(); - uploadRequest.addProperty("fileName", filePath); - + uploadRequest.addProperty("fileName", localFile.getFileName().toString()); + uploadRequest.addProperty("contentLength", localFile.toFile().length()); + log.debug(" POST {}", url); HttpRequest request = HttpRequest.newBuilder() .uri(URI.create(url)) .header("Content-Type", "application/json") @@ -276,34 +277,49 @@ protected void uploadFile(String filePath, Path localFile) throws IOException { HttpResponse response = httpClient.sendAsString(request); if (response.statusCode() >= 400) { - throw new IOException("Failed to get upload URL for: " + filePath + " - status: " + response.statusCode()); + String message = response.body(); + if (message == null) message = ""; + message = message + " - HTTP " + response.statusCode(); + throw new IOException("Failed to get upload URL for '" + filePath + "': " + message); } - + log.debug(response.body()); JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); - String uploadUrl = responseJson.get("uploadUrl").getAsString(); - - // Step 2: Upload file to the provided URL - HttpRequest uploadFileRequest = HttpRequest.newBuilder() - .uri(URI.create(uploadUrl)) - .header("Content-Type", "application/octet-stream") - .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) - .build(); - - HttpResponse uploadResponse = httpClient.send(uploadFileRequest, HttpResponse.BodyHandlers.discarding()); + String uploadId = responseJson.get("uploadId").getAsString(); + JsonArray jsonUploadUrls = responseJson.getAsJsonArray("uploadUrls").getAsJsonArray(); + List uploadUrls = StreamSupport.stream(jsonUploadUrls.spliterator(), false) + .map(JsonElement::getAsString) + .collect(java.util.stream.Collectors.toList()); - if (uploadResponse.statusCode() >= 400) { - throw new IOException("Failed to upload file: " + filePath + " - status: " + uploadResponse.statusCode()); + for (String uploadUrl : uploadUrls) { + // Step 2: Upload file to the provided URL + HttpRequest uploadFileRequest = HttpRequest.newBuilder() + .uri(URI.create(uploadUrl)) + .header("Content-Type", "application/octet-stream") + .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) + .build(); + + HttpResponse uploadResponse = httpClient.send(uploadFileRequest, HttpResponse.BodyHandlers.discarding()); + + if (uploadResponse.statusCode() >= 400) { + String message = response.body(); + if (message == null) message = ""; + message = message + " - HTTP " + response.statusCode(); + throw new IOException("Failed to upload file '" + filePath + "': " + message); + } } - // Step 3: Finish upload (required for some providers like AWS) - - String finishUrl = buildDataLinkUrl("/upload/finish", Map.of()); + queryParams = new HashMap<>(); if (workspaceId != null) { - finishUrl += "?workspaceId=" + workspaceId; + queryParams.put("workspaceId", workspaceId); + } + if (dataLink.credentialsId != null) { + queryParams.put("credentialsId", dataLink.credentialsId); } + String finishUrl = buildDataLinkUrl("/upload/finish/"+ URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); JsonObject finishRequest = new JsonObject(); finishRequest.addProperty("fileName", filePath); + finishRequest.addProperty("uploadId", filePath); HttpRequest finishReq = HttpRequest.newBuilder() .uri(URI.create(finishUrl)) @@ -350,7 +366,7 @@ protected List listFiles(String path) throws IOException { if (message == null) message = ""; message = message + " - HTTP " + response.statusCode(); - throw new IOException("Failed to list files in: " + path + message); + throw new IOException("Failed to list files in '" + path + "' : " + message); } log.debug(response.body()); JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); From b1e6c4abe36a3de913b6308d5d3257d7a31b7c9f Mon Sep 17 00:00:00 2001 From: jorgee Date: Mon, 3 Nov 2025 09:41:06 +0100 Subject: [PATCH 3/4] Add data-link seqera fs and jgit transport and including push in launch command Signed-off-by: jorgee --- .../groovy/nextflow/scm/AssetManager.groovy | 7 +- .../src/main/nextflow/file/FileHelper.groovy | 2 +- plugins/nf-tower/build.gradle | 3 +- .../io/seqera/tower/plugin/TowerClient.groovy | 52 +- .../tower/plugin/TowerHxClientFactory.groovy | 94 ++ .../tower/plugin/TowerRetryPolicy.groovy | 2 +- .../tower/plugin/datalink/DataLink.groovy | 34 + .../plugin/datalink/DataLinkUtils.groovy | 836 ++++++++++++++++++ .../tower/plugin/fs/SeqeraFileSystem.java | 173 ++++ .../plugin/fs/SeqeraFileSystemProvider.java | 724 +++++++++++++++ .../io/seqera/tower/plugin/fs/SeqeraPath.java | 366 ++++++++ .../tower/plugin/fs/SeqeraPathFactory.java | 68 ++ .../plugin/launch/LaunchCommandImpl.groovy | 54 +- .../tower/plugin/launch/PushManager.groovy | 416 +++++++++ .../plugin/scm/SeqeraProviderConfig.groovy | 23 +- .../scm/SeqeraRepositoryProvider.groovy | 3 + .../plugin/scm/jgit/SeqeraBaseConnection.java | 371 +------- .../scm/jgit/SeqeraFetchConnection.java | 7 +- .../jgit/SeqeraGitCredentialsProvider.java | 6 + .../plugin/scm/jgit/SeqeraPushConnection.java | 14 +- .../java.nio.file.spi.FileSystemProvider | 1 + 21 files changed, 2815 insertions(+), 441 deletions(-) create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerHxClientFactory.groovy create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLink.groovy create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLinkUtils.groovy create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystem.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystemProvider.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPath.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPathFactory.java create mode 100644 plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy create mode 100644 plugins/nf-tower/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider diff --git a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy index 047d162f1c..c84c65b465 100644 --- a/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/AssetManager.groovy @@ -328,10 +328,13 @@ class AssetManager { String getHub() { hub } + static boolean isUrl(String repository){ + return repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') || repository.startsWith('seqera://') + } + @PackageScope String resolveNameFromGitUrl( String repository ) { - final isUrl = repository.startsWith('http://') || repository.startsWith('https://') || repository.startsWith('file:/') || repository.startsWith('seqera://') - if( !isUrl ) + if( !isUrl(repository) ) return null try { diff --git a/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy b/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy index 83df9c6034..35cdee5aad 100644 --- a/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy +++ b/modules/nf-commons/src/main/nextflow/file/FileHelper.groovy @@ -362,7 +362,7 @@ class FileHelper { return asPath(toPathURI(str)) } - static final private Map PLUGINS_MAP = [s3:'nf-amazon', gs:'nf-google', az:'nf-azure'] + static final private Map PLUGINS_MAP = [s3:'nf-amazon', gs:'nf-google', az:'nf-azure', seqera: 'nf-tower'] static final private Map SCHEME_CHECKED = new HashMap<>() diff --git a/plugins/nf-tower/build.gradle b/plugins/nf-tower/build.gradle index 460097f92f..e64b114ddc 100644 --- a/plugins/nf-tower/build.gradle +++ b/plugins/nf-tower/build.gradle @@ -27,7 +27,8 @@ nextflowPlugin { 'io.seqera.tower.plugin.TowerFusionToken', 'io.seqera.tower.plugin.auth.AuthCommandImpl', 'io.seqera.tower.plugin.launch.LaunchCommandImpl', - 'io.seqera.tower.plugin.scm.SeqeraRepositoryFactory' + 'io.seqera.tower.plugin.scm.SeqeraRepositoryFactory', + 'io.seqera.tower.plugin.fs.SeqeraPathFactory' ] } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerClient.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerClient.groovy index d070b758c8..35c47a6198 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerClient.groovy +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerClient.groovy @@ -70,8 +70,6 @@ class TowerClient implements TraceObserverV2 { static private final Duration ALIVE_INTERVAL = Duration.of('1 min') - static private final String TOKEN_PREFIX = '@token:' - @TupleConstructor static class Response { final int code @@ -274,8 +272,7 @@ class TowerClient implements TraceObserverV2 { this.aggregator = new ResourcesAggregator(session) this.runName = session.getRunName() this.runId = session.getUniqueId() - this.httpClient = newHttpClient() - + this.httpClient = TowerHxClientFactory.httpClient(getAccessToken(), env.get('TOWER_REFRESH_TOKEN'), this.endpoint, this.retryPolicy) // send hello to verify auth final req = makeCreateReq(session) final resp = sendHttpMessage(urlTraceCreate, req, 'POST') @@ -299,53 +296,6 @@ class TowerClient implements TraceObserverV2 { reports.flowCreate(workflowId) } - protected HxClient newHttpClient() { - final builder = HxClient.newBuilder() - // auth settings - setupClientAuth(builder, getAccessToken()) - // retry settings - builder - .retryConfig(this.retryPolicy) - .followRedirects(HttpClient.Redirect.NORMAL) - .version(HttpClient.Version.HTTP_1_1) - .connectTimeout(java.time.Duration.ofSeconds(60)) - .build() - } - - protected void setupClientAuth(HxClient.Builder config, String token) { - // check for plain jwt token - final refreshToken = env.get('TOWER_REFRESH_TOKEN') - final refreshUrl = refreshToken ? "$endpoint/oauth/access_token" : null - if( token.count('.')==2 ) { - config.bearerToken(token) - config.refreshToken(refreshToken) - config.refreshTokenUrl(refreshUrl) - config.refreshCookiePolicy(CookiePolicy.ACCEPT_ALL) - return - } - - // try checking personal access token - try { - final plain = new String(token.decodeBase64()) - final p = plain.indexOf('.') - if( p!=-1 && new JsonSlurper().parseText( plain.substring(0, p) ) ) { - // ok this is bearer token - config.bearerToken(token) - // setup the refresh - config.refreshToken(refreshToken) - config.refreshTokenUrl(refreshUrl) - config.refreshCookiePolicy(CookiePolicy.ACCEPT_ALL) - return - } - } - catch ( Exception e ) { - log.trace "Enable to set bearer token ~ Reason: $e.message" - } - - // fallback on simple token - config.basicAuth(TOKEN_PREFIX + token) - } - protected Map makeCreateReq(Session session) { def result = new HashMap(5) result.sessionId = session.uniqueId.toString() diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerHxClientFactory.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerHxClientFactory.groovy new file mode 100644 index 0000000000..e339eeee07 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerHxClientFactory.groovy @@ -0,0 +1,94 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package io.seqera.tower.plugin + +import groovy.json.JsonSlurper +import groovy.transform.CompileStatic +import groovy.transform.Memoized +import groovy.util.logging.Slf4j +import io.seqera.http.HxClient +import nextflow.Global +import nextflow.Session +import nextflow.SysEnv +import nextflow.file.http.XAuthProvider +import nextflow.file.http.XAuthRegistry +import nextflow.trace.TraceObserverV2 +import nextflow.util.Duration + +import java.net.http.HttpClient + +/** + * Create and register the Tower observer instance + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class TowerHxClientFactory { + + static private final String TOKEN_PREFIX = '@token:' + + @Memoized + static HxClient httpClient(String accessToken, String refreshToken, String endpoint, TowerRetryPolicy retryPolicy) { + assert accessToken + final builder = HxClient.newBuilder() + // auth settings + setupClientAuth(builder, accessToken, refreshToken, endpoint) + // retry settings + builder + .retryConfig(retryPolicy) + .followRedirects(HttpClient.Redirect.NORMAL) + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(java.time.Duration.ofSeconds(60)) + .build() + } + + private static void setupClientAuth(HxClient.Builder config, String token, String refreshToken, String endpoint) { + // check for plain jwt token + + final refreshUrl = refreshToken ? "$endpoint/oauth/access_token" : null + if( token.count('.')==2 ) { + config.bearerToken(token) + config.refreshToken(refreshToken) + config.refreshTokenUrl(refreshUrl) + config.refreshCookiePolicy(CookiePolicy.ACCEPT_ALL) + return + } + + // try checking personal access token + try { + final plain = new String(token.decodeBase64()) + final p = plain.indexOf('.') + if( p!=-1 && new JsonSlurper().parseText( plain.substring(0, p) ) ) { + // ok this is bearer token + config.bearerToken(token) + // setup the refresh + config.refreshToken(refreshToken) + config.refreshTokenUrl(refreshUrl) + config.refreshCookiePolicy(CookiePolicy.ACCEPT_ALL) + return + } + } + catch ( Exception e ) { + log.trace "Enable to set bearer token ~ Reason: $e.message" + } + + // fallback on simple token + config.basicAuth(TOKEN_PREFIX + token) + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerRetryPolicy.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerRetryPolicy.groovy index 076bf3df06..255e2c13de 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerRetryPolicy.groovy +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/TowerRetryPolicy.groovy @@ -75,7 +75,7 @@ class TowerRetryPolicy implements Retryable.Config, ConfigScope { TowerRetryPolicy(Map opts, Map legacy=Map.of()) { this.delay = opts.delay as Duration ?: legacy.backOffDelay as Duration ?: RetryConfig.DEFAULT_DELAY this.maxDelay = opts.maxDelay as Duration ?: RetryConfig.DEFAULT_MAX_DELAY - this.maxAttempts = opts.maxAttemps as Integer ?: legacy.maxRetries as Integer ?: RetryConfig.DEFAULT_MAX_ATTEMPTS + this.maxAttempts = opts.maxAttempts as Integer ?: legacy.maxRetries as Integer ?: RetryConfig.DEFAULT_MAX_ATTEMPTS this.jitter = opts.jitter as Double ?: RetryConfig.DEFAULT_JITTER this.multiplier = opts.multiplier as Double ?: legacy.backOffBase as Double ?: RetryConfig.DEFAULT_MULTIPLIER } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLink.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLink.groovy new file mode 100644 index 0000000000..0fc3c7854e --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLink.groovy @@ -0,0 +1,34 @@ +package io.seqera.tower.plugin.datalink + +import groovy.transform.CompileStatic + +@CompileStatic +class DataLink { + private String name + private String id + private String credentialsId + private String provider + + DataLink(String name, String id, String credentialsId, String provider) { + this.name = name + this.id = id + this.credentialsId = credentialsId + this.provider = provider + } + String getName(){ + return name + } + + String getId() { + return id + } + + String getCredentialsId() { + return credentialsId + } + + String getProvider() { + return provider + } + +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLinkUtils.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLinkUtils.groovy new file mode 100644 index 0000000000..b80bcbe47a --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/datalink/DataLinkUtils.groovy @@ -0,0 +1,836 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.datalink + +import groovy.json.JsonBuilder +import groovy.json.JsonSlurper +import groovy.transform.Canonical +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import io.seqera.http.HxClient + +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.time.Duration + +/** + * Utility class for data-link file transfers (upload/download). + * Implements cloud provider-specific upload strategies based on tower-cli patterns. + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +class DataLinkUtils { + + private static final JsonSlurper jsonSlurper = new JsonSlurper() + private static final int CHUNK_SIZE = 250 * 1024 * 1024 // 250 MB chunks for multipart uploads + private static final int DOWNLOAD_BUFFER_SIZE = 8192 // 8 KB buffer for downloads + + /** + * Get file content as byte array from a data-link + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link associated to the file + * @param filePath Remote file path + * @param workspaceId Workspace ID (optional) + * @return File content as byte array, or null if file not found + * @throws IOException if download fails + */ + static byte[] getFileContent(HxClient httpClient, String endpoint, DataLink dataLink, + String filePath, String workspaceId) throws IOException { + try { + final credentialsId = dataLink.getCredentialsId() + final dataLinkId = dataLink.getId() + + def queryParams = [:] as Map + if (workspaceId) { + queryParams.workspaceId = workspaceId + } + if (credentialsId) { + queryParams.credentialsId = credentialsId + } + + def url = buildDataLinkUrl(endpoint, dataLinkId, + "/download/${URLEncoder.encode(filePath, StandardCharsets.UTF_8)}", queryParams) + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build() + + def response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) + + if (response.statusCode() == 404) { + log.debug("File {} in data-link {} not found", filePath, dataLinkId) + return null + } + + if (response.statusCode() >= 400) { + throw new IOException("Failed to download file: ${filePath} - status: ${response.statusCode()}") + } + + return response.body() + + } catch (InterruptedException e) { + Thread.currentThread().interrupt() + throw new IOException("Download interrupted", e) + } + } + + /** + * Download a file from a data-link using streaming to avoid memory issues + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link associated to the file + * @param filePath Remote file path + * @param targetPath Local target path + * @param workspaceId Workspace ID (optional) + * @param credentialsId Credentials ID (optional) + * @throws IOException if download fails + */ + static void downloadFile(HxClient httpClient, String endpoint, DataLink dataLink, + String filePath, Path targetPath, + String workspaceId) throws IOException { + try { + final credentialsId = dataLink.getCredentialsId() + final dataLinkId = dataLink.getId() + // Build URL for getting download URL + def queryParams = [:] as Map + if (workspaceId) { + queryParams.workspaceId = workspaceId + } + if (credentialsId) { + queryParams.credentialsId = credentialsId + } + queryParams.filePath = filePath + queryParams.preview = 'false' + + def url = buildDataLinkUrl(endpoint, dataLinkId, "/generate-download-url", queryParams) + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build() + + def response = httpClient.sendAsString(request) + log.debug("Upload response: {}", response.body()) + if (response.statusCode() == 404) { + log.debug("File {} in data-link {} not found", filePath, dataLinkId) + throw new IOException("File not found: ${filePath}") + } + + if (response.statusCode() >= 400) { + throw new IOException("Failed to get download URL for file: ${filePath} - status: ${response.statusCode()}") + } + + // Parse response to get download URL + def responseJson = jsonSlurper.parseText(response.body()) as Map + def downloadUrl = responseJson.url as String + + // Download file using streaming + log.debug("Downloading {} from {}", filePath, downloadUrl) + downloadFileFromUrl(downloadUrl, targetPath) + + } catch (InterruptedException e) { + Thread.currentThread().interrupt() + throw new IOException("Download interrupted", e) + } + } + + /** + * Download a file from a direct URL using streaming + */ + private static void downloadFileFromUrl(String url, Path targetPath) throws IOException, InterruptedException { + HxClient httpClient = HxClient.newHxClient() + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build() + + def response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()) + + if (response.statusCode() != 200) { + throw new IOException("Failed to download file: HTTP ${response.statusCode()}") + } + + // Stream the download to avoid loading entire file in memory + response.body().withCloseable { input -> + Files.newOutputStream(targetPath).withCloseable { output -> + def buffer = new byte[DOWNLOAD_BUFFER_SIZE] + int bytesRead + while ((bytesRead = input.read(buffer)) != -1) { + output.write(buffer, 0, bytesRead) + } + } + } + + log.debug("Downloaded to {}", targetPath) + } + + /** + * Upload a file to a data-link with provider-specific strategies + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLinkId Data-link ID + * @param filePath Remote file path + * @param localFile Local file to upload + * @param provider Cloud provider type + * @param workspaceId Workspace ID (optional) + * @param credentialsId Credentials ID (optional) + * @throws IOException if upload fails + */ + static void uploadFile(HxClient httpClient, String endpoint, DataLink dataLink, + String filePath, Path localFile, + String workspaceId) throws IOException { + try { + if (!workspaceId) + throw new IOException("Workspace must be specified for uploading files to data links") + final credentialsId = dataLink.getCredentialsId() + final provider = dataLink.getProvider() + final dataLinkId = dataLink.getId() + // Step 1: Get upload URLs + final queryParams = [:] as Map + queryParams.workspaceId = workspaceId + if (credentialsId) { + queryParams.credentialsId = credentialsId + } + + def url = buildDataLinkUrl(endpoint, dataLinkId, "/upload", queryParams) + // Detect MIME type + def mimeType = Files.probeContentType(localFile) ?: 'application/octet-stream' + + def uploadRequest = [ + fileName: filePath, + contentLength: localFile.toFile().length(), + contentType: mimeType + ] + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header('Content-Type', 'application/json') + .POST(HttpRequest.BodyPublishers.ofString(new JsonBuilder(uploadRequest).toString())) + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() >= 400) { + def message = response.body() ?: '' + message = "${message} - HTTP ${response.statusCode()}" + throw new IOException("Failed to get upload URL for '${filePath}': ${message}") + } + + log.debug("Upload response: {}", response.body()) + def responseJson = jsonSlurper.parseText(response.body()) as Map + def uploadId = responseJson.uploadId as String + def uploadUrls = responseJson.uploadUrls as List + + // Step 2: Upload file using provider-specific strategy + def etags = [] as List + def withError = false + + try { + if (provider?.equalsIgnoreCase('AWS') || provider?.equalsIgnoreCase('SEQERACOMPUTE')) { + etags = uploadFileAws( localFile.toFile(), uploadUrls) + } else if (provider?.equalsIgnoreCase('GOOGLE')) { + uploadFileGoogle( localFile.toFile(), uploadUrls) + } else if (provider?.equalsIgnoreCase('AZURE')) { + uploadFileAzure( localFile.toFile(), uploadUrls) + } else { + throw new IOException("Unsupported provider: ${provider}") + } + } catch (Exception e) { + withError = true + log.error("Upload failed: {}", e.message, e) + throw new IOException("Failed to upload file: ${e.message}", e) + } finally { + // Step 3: Finalize upload + finalizeUpload(httpClient, endpoint, dataLinkId, filePath, uploadId, etags, withError, workspaceId, credentialsId) + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt() + throw new IOException("Upload interrupted", e) + } + } + + /** + * Upload an entire folder (directory) to a data-link recursively + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link to upload to + * @param remotePath Remote folder path in data-link + * @param localFolder Local folder to upload + * @param workspaceId Workspace ID + * @throws IOException if upload fails + */ + static void uploadFolder(HxClient httpClient, String endpoint, DataLink dataLink, + String remotePath, Path localFolder, + String workspaceId) throws IOException { + if (!Files.isDirectory(localFolder)) { + throw new IOException("Local path is not a directory: ${localFolder}") + } + + log.debug("Uploading folder {} to {}", localFolder, remotePath) + + // Walk through the directory tree + Files.walk(localFolder).each { Path localFile -> + if (Files.isRegularFile(localFile)) { + // Calculate relative path + def relativePath = localFolder.relativize(localFile).toString() + // Normalize path separators to forward slashes for remote path + relativePath = relativePath.replace('\\', '/') + // Combine with remote path + def remoteFilePath = (remotePath ? "${remotePath}/${relativePath}" : relativePath) as String + + log.debug("Uploading file {} to {}", localFile, remoteFilePath) + uploadFile(httpClient, endpoint, dataLink, remoteFilePath, localFile, workspaceId) + } + } + + log.debug("Folder upload completed: {}", remotePath) + } + + /** + * Download an entire folder (directory) from a data-link recursively + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link to download from + * @param remotePath Remote folder path in data-link + * @param localFolder Local folder to download to + * @param workspaceId Workspace ID (optional) + * @throws IOException if download fails + */ + static void downloadFolder(HxClient httpClient, String endpoint, DataLink dataLink, + String remotePath, Path localFolder, + String workspaceId) throws IOException { + log.debug("Downloading folder {} to {}", remotePath, localFolder) + + // Create local folder if it doesn't exist + if (!Files.exists(localFolder)) { + Files.createDirectories(localFolder) + } + + // List all files in the remote folder + def items = listFiles(httpClient, endpoint, dataLink, remotePath, workspaceId) + + for (String itemName : items) { + // Calculate full remote path + def remoteItemPath = (remotePath ? "${remotePath}/${itemName}" : itemName) as String + def localItemPath = localFolder.resolve(itemName.replaceAll('/$', '')) + + // Check if item is a folder (ends with /) + if (itemName.endsWith('/')) { + // It's a folder, recurse + log.debug("Downloading subfolder {}", remoteItemPath) + def remoteFolderPath = remoteItemPath.replaceAll('/$', '') as String + downloadFolder(httpClient, endpoint, dataLink, remoteFolderPath, localItemPath, workspaceId) + } else { + // It's a file, download it + log.debug("Downloading file {} to {}", remoteItemPath, localItemPath) + downloadFile(httpClient, endpoint, dataLink, remoteItemPath, localItemPath, workspaceId) + } + } + + log.debug("Folder download completed: {}", remotePath) + } + + /** + * Upload file using AWS S3 multipart upload protocol + */ + private static List uploadFileAws( File file, List uploadUrls) throws IOException, InterruptedException { + def etags = [] as List + int index = 0 + + HxClient httpClient = HxClient.newHxClient() + for (String url : uploadUrls) { + log.debug("PUT chunk $index in $url") + def chunk = getChunk(file, index) + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header('Content-Type', 'application/octet-stream') + .PUT(HttpRequest.BodyPublishers.ofByteArray(chunk)) + .build() + + def response = httpClient.sendAsString(request) + log.debug("Response: $response") + if (response.statusCode() != 200) { + log.error("Failed to upload chunk ${index}: HTTP ${response.statusCode()}, Message: ${response.body()}") + throw new IOException("Failed to upload chunk ${index}: HTTP ${response.statusCode()}, Message: ${response.body()}") + } + + // Extract ETag from response + def etag = response.headers().firstValue('ETag').orElse(null) + if (etag) { + etags << etag + } else { + throw new IOException("Failed to get ETag from upload response - possible CORS issue") + } + log.debug("Chunk $index uploaded.") + index++ + } + + return etags + } + + /** + * Upload file using Google Cloud Storage resumable upload protocol + */ + private static void uploadFileGoogle( File file, List uploadUrls) throws IOException, InterruptedException { + if (!uploadUrls) { + throw new IOException('No upload URLs provided') + } + + def url = uploadUrls[0] // Google uses single resumable upload URL + def fileSize = file.length() + int index = 0 + long bytesUploaded = 0 + HxClient httpClient = HxClient.newHxClient() + while (bytesUploaded < fileSize) { + def chunk = getChunk(file, index) + def start = bytesUploaded + def end = Math.min(start + chunk.length, fileSize) - 1 + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header('Content-Type', 'application/octet-stream') + .header('Content-Range', "bytes ${start}-${end}/${fileSize}") + .PUT(HttpRequest.BodyPublishers.ofByteArray(chunk)) + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() == 308) { + // Resume incomplete - continue with next chunk + def range = response.headers().firstValue('Range').orElse('') + log.debug("Upload progress: {}", range) + } else if (response.statusCode() == 200 || response.statusCode() == 201) { + // Upload complete + break + } else { + throw new IOException("Failed to upload chunk ${index}: HTTP ${response.statusCode()}") + } + + bytesUploaded += chunk.length + index++ + } + } + + /** + * Upload file using Azure Block Blob protocol + */ + private static void uploadFileAzure(File file, List uploadUrls) throws IOException, InterruptedException { + def blockIds = [] as List + int index = 0 + HxClient httpClient = HxClient.newHxClient() + for (String url : uploadUrls) { + def chunk = getChunk(file, index) + + // Extract block ID from URL (typically in comp=block&blockid=XXX) + def blockId = extractBlockId(url, index) + blockIds << blockId + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header('Content-Type', 'application/octet-stream') + .PUT(HttpRequest.BodyPublishers.ofByteArray(chunk)) + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() != 201 && response.statusCode() != 200) { + throw new IOException("Failed to upload block ${index}: HTTP ${response.statusCode()}") + } + + index++ + } + } + + /** + * Extract block ID from Azure upload URL + */ + private static String extractBlockId(String url, int index) { + // Try to extract from URL parameter + def blockIdIdx = url.indexOf('blockid=') + if (blockIdIdx != -1) { + def endIdx = url.indexOf('&', blockIdIdx) + if (endIdx == -1) endIdx = url.length() + return url.substring(blockIdIdx + 8, endIdx) + } + // Fallback: generate block ID + return String.format('block-%05d', index) + } + + /** + * Get a chunk of the file for multipart upload + */ + private static byte[] getChunk(File file, int index) throws IOException { + new RandomAccessFile(file, 'r').withCloseable { raf -> + def start = (long) index * CHUNK_SIZE + def end = Math.min(start + CHUNK_SIZE, file.length()) + def length = (int) (end - start) + + def buffer = new byte[length] + raf.seek(start) + raf.readFully(buffer) + + return buffer + } + } + + /** + * Finalize the upload by calling the finish endpoint + */ + private static void finalizeUpload(HxClient httpClient, String endpoint, String dataLinkId, + String filePath, String uploadId, List etags, + boolean withError, String workspaceId, String credentialsId) + throws IOException, InterruptedException { + + def queryParams = [:] as Map + if (workspaceId) { + queryParams.workspaceId = workspaceId + } + if (credentialsId) { + queryParams.credentialsId = credentialsId + } + + def url = buildDataLinkUrl(endpoint, dataLinkId, "/upload/finish", queryParams) + + def finishRequest = [ + fileName: filePath, + uploadId: uploadId, + withError: withError + ] as Map + + // Add ETags if present (for AWS) + if (etags) { + def tags = etags.withIndex().collect { String etag, int i -> + [ + eTag: etag, + partNumber: i + 1 + ] as Map + } + finishRequest.tags = tags + } + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header('Content-Type', 'application/json') + .POST(HttpRequest.BodyPublishers.ofString(new JsonBuilder(finishRequest).toString())) + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() >= 400) { + log.warn("Failed to finalize upload: HTTP {}, body: {}", response.statusCode(), response.body()) + } else { + log.debug("Upload finalized successfully") + } + } + + /** + * Build data-link URL with path and query parameters + */ + private static String buildDataLinkUrl(String endpoint, String dataLinkId, String path, Map queryParams) { + def url = new StringBuilder("${endpoint}/data-links/${URLEncoder.encode(dataLinkId, StandardCharsets.UTF_8)}") + if (path) { + if (!path.startsWith('/')) url.append('/') + url.append(path) + } + + if (queryParams) { + def queryString = queryParams.collect { key, value -> + "${URLEncoder.encode(key, StandardCharsets.UTF_8)}=${URLEncoder.encode(value, StandardCharsets.UTF_8)}" + }.join('&') + url.append('?').append(queryString) + } + return url.toString() + } + + /** + * Find DatalinkId from the bucket name + */ + static DataLink findDataLink(HxClient httpClient, String endpoint, String workspaceId, String linkName) { + try { + + String url = endpoint + "/data-links?status=AVAILABLE&search=" + linkName + if( workspaceId != null ) { + url += "&workspaceId=" + workspaceId + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build() + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()) + + if( response.statusCode() < 200 && response.statusCode() >= 300 ) { + String message = response.body() + if( message == null ) message = "" + message = message + " - HTTP " + response.statusCode() + throw new RuntimeException("Failed to find data-link: " + linkName + message) + } + + def responseJson = jsonSlurper.parseText(response.body()) as Map + def dataLinksArray = responseJson.dataLinks as List + + List dataLinks = dataLinksArray + .findAll { it.name == linkName } + .collect { getDataLinkFromJSON(linkName, it) } + + if( dataLinks.isEmpty() ) { + log.debug("No datalink response: " + linkName) + throw new RuntimeException("No Data-link found for " + linkName) + } + return dataLinks.first() + } catch( IOException e ) { + throw new RuntimeException("Exception finding data-link", e) + } catch( InterruptedException e ) { + Thread.currentThread().interrupt() + throw new RuntimeException("Download interrupted", e) + } + } + + private static DataLink getDataLinkFromJSON(String linkName, Map obj) { + String id = obj.id as String + //Get first credential + def credentials = obj.credentials as List + String credentialsId = credentials[0].id as String + //Get provider type (AWS, GOOGLE, AZURE, etc.) + String provider = obj.provider ? obj.provider as String : "AWS" + + return new DataLink(linkName, id, credentialsId, provider) + } + + /** + * List files in a path within a data-link + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link to browse + * @param path Path within the data-link to list + * @param workspaceId Workspace ID (optional) + * @return List of file/directory names in the specified path + * @throws IOException if listing fails + */ + static List listFiles(HxClient httpClient, String endpoint, DataLink dataLink, + String path, String workspaceId) throws IOException { + try { + final queryParams = [:] as Map + if (workspaceId) { + queryParams.workspaceId = workspaceId + } + if (dataLink.getCredentialsId()) { + queryParams.credentialsId = dataLink.getCredentialsId() + } + path = path ?: '' + def url = buildDataLinkUrl(endpoint, dataLink.getId(), + "/browse/${URLEncoder.encode(path, StandardCharsets.UTF_8)}", queryParams) + log.debug("GET {}", url) + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() == 404) { + log.debug("No files found for {}", url) + return [] + } + + if (response.statusCode() >= 400) { + log.debug("Error getting files {}", url) + def message = response.body() ?: '' + message = "${message} - HTTP ${response.statusCode()}" + throw new IOException("Failed to list files in '${path}': ${message}") + } + + def responseJson = jsonSlurper.parseText(response.body()) as Map + def files = responseJson.objects as List + + return files.collect { it.name as String } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt() + throw new IOException("List files interrupted", e) + } + } + + /** + * Get the details of a path within a data-link + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link to browse + * @param path Path within the data-link to get the details + * @param workspaceId Workspace ID (optional) + * @return Detail of the file/directory in the specified path. Null if file is not found + * @throws IOException if listing fails + */ + static DataLinkItem getFileDetails(HxClient httpClient, String endpoint, DataLink dataLink, + String pathStr, String workspaceId) throws IOException { + try { + if(!pathStr) + return new DataLinkItem(DataLinkItemType.FOLDER, dataLink.name, 0, null) + + def path = Path.of(pathStr) + def fileName = path.getFileName()?.toString() + def parent = path.getParent()?.toString() ?: '' + final queryParams = [:] as Map + if (workspaceId) { + queryParams.workspaceId = workspaceId + } + if (dataLink.getCredentialsId()) { + queryParams.credentialsId = dataLink.getCredentialsId() + } + queryParams.search = fileName + + def url = buildDataLinkUrl(endpoint, dataLink.getId(), + "/browse/${URLEncoder.encode(parent, StandardCharsets.UTF_8)}", queryParams) + log.debug("GET {}", url) + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .GET() + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() == 404) { + log.debug("No files found for {}", url) + return null + } + + if (response.statusCode() >= 400) { + log.debug("Error getting files {}", url) + def message = response.body() ?: '' + message = "${message} - HTTP ${response.statusCode()}" + throw new IOException("Failed to get file in '${path}': ${message}") + } + + def responseJson = jsonSlurper.parseText(response.body()) as Map + def files = responseJson.objects as List + if (files){ + files = files.stream().filter {it.type == 'FOLDER' ? it.name == fileName + '/' : it.name == fileName } .toList() + } + if (files.isEmpty()) + return null + if (files.size() > 1) { + throw new IOException("Two items with the same name $fileName") + } + Map item = files.get(0) + return new DataLinkItem(DataLinkItemType.valueOf(item.type as String), item.name as String, item.size as Integer, item.mimeType as String) + + } catch (InterruptedException e) { + Thread.currentThread().interrupt() + throw new IOException("List files interrupted", e) + } + } + + /** + * Delete a file from a data-link + * + * @param httpClient HTTP client to use + * @param endpoint Platform API endpoint + * @param dataLink Data-link containing the file + * @param filePath Path of file to delete + * @param workspaceId Workspace ID (optional) + * @throws IOException if delete fails + */ + static void deleteFile(HxClient httpClient, String endpoint, DataLink dataLink, + String filePath, String workspaceId) throws IOException { + try { + final queryParams = [:] as Map + if (workspaceId) { + queryParams.workspaceId = workspaceId + } + if (dataLink.getCredentialsId()) { + queryParams.credentialsId = dataLink.getCredentialsId() + } + + def url = buildDataLinkUrl(endpoint, dataLink.getId(), '/content', queryParams) + + def deleteRequest = [ + files: [filePath] + ] + + def request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .header('Content-Type', 'application/json') + .method('DELETE', HttpRequest.BodyPublishers.ofString(new JsonBuilder(deleteRequest).toString())) + .build() + + def response = httpClient.sendAsString(request) + + if (response.statusCode() >= 400) { + log.debug("Failed to delete file {}", url) + def message = response.body() ?: '' + message = "${message} - HTTP ${response.statusCode()}" + throw new IOException("Failed to delete file ${filePath}: ${message}") + } + + def responseJson = jsonSlurper.parseText(response.body()) as Map + if (responseJson.deletionFailures) { + def failures = responseJson.deletionFailures as List + if (failures) { + throw new IOException("Detected deletion failures: ${parseFailures(failures)}") + } + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt() + throw new IOException("Delete interrupted", e) + } + } + + private static String parseFailures(List failures) { + def message = [] + for (Map failure : failures ){ + final error = failure.errorMessage as String + final item = failure.dataLinkItem as Map + message.add("${item.type} ${item.name}: ${error}") + } + return message.join(',') + } + + @Canonical + static class DataLinkItem { + DataLinkItemType type + String name + Integer size + String mimeType + } + + enum DataLinkItemType { + FOLDER, + FILE, + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystem.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystem.java new file mode 100644 index 0000000000..43780608e7 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystem.java @@ -0,0 +1,173 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.fs; + +import com.google.common.collect.ImmutableSet; +import io.seqera.http.HxClient; +import io.seqera.tower.plugin.datalink.DataLink; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.nio.file.spi.FileSystemProvider; +import java.util.Collections; +import java.util.Objects; +import java.util.Set; + +/** + * File system for Seqera Platform Data-Link Paths + * + * @author Jorge Ejarque + */ +public class SeqeraFileSystem extends FileSystem { + + private final SeqeraFileSystemProvider provider; + private final String endpoint; + private final String workspaceId; + private final HxClient httpClient; + private final DataLink dataLink; + + /* + * Only needed to prevent serialization issues - see https://github.com/nextflow-io/nextflow/issues/5208 + */ + protected SeqeraFileSystem() { + this.provider = null; + this.endpoint = null; + this.workspaceId = null; + this.httpClient = null; + this.dataLink = null; + } + + public SeqeraFileSystem(SeqeraFileSystemProvider provider, DataLink dataLink, String endpoint, String workspaceId, HxClient httpClient) { + this.provider = provider; + this.endpoint = endpoint; + this.workspaceId = workspaceId; + this.httpClient = httpClient; + this.dataLink = dataLink; + } + + public String getEndpoint() { + return endpoint; + } + + public String getWorkspaceId() { + return workspaceId; + } + + public DataLink getDataLink() { + return dataLink; + } + + public String getDataLinkName() { + return dataLink != null ? dataLink.getName() : null; + } + + public HxClient getHttpClient() { + return httpClient; + } + + @Override + public boolean equals(Object other) { + if (this.getClass() != other.getClass()) { + return false; + } + final SeqeraFileSystem that = (SeqeraFileSystem) other; + return Objects.equals(this.provider, that.provider) && + Objects.equals(this.dataLink.getId(), that.dataLink.getId()) && + Objects.equals(this.endpoint, that.endpoint); + } + + @Override + public int hashCode() { + return Objects.hash(provider, dataLink.getId(), endpoint); + } + + @Override + public FileSystemProvider provider() { + return provider; + } + + @Override + public void close() throws IOException { + // Cleanup if needed + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public boolean isReadOnly() { + return false; // Allow both read and write operations + } + + @Override + public String getSeparator() { + return SeqeraPath.SEPARATOR; + } + + @Override + public Iterable getRootDirectories() { + return Collections.singletonList(getPath("/")); + } + + @Override + public Iterable getFileStores() { + return null; + } + + @Override + public Set supportedFileAttributeViews() { + return ImmutableSet.of("basic"); + } + + @Override + public Path getPath(String first, String... more) { + final StringBuilder pathBuilder = new StringBuilder(first); + if (more != null && more.length > 0) { + for (String segment : more) { + pathBuilder.append(SeqeraPath.SEPARATOR).append(segment); + } + } + return new SeqeraPath(this, pathBuilder.toString()); + } + + public Path getPath(URI uri) { + return new SeqeraPath(this, uri); + } + + @Override + public PathMatcher getPathMatcher(String syntaxAndPattern) { + throw new UnsupportedOperationException("Path matcher not supported"); + } + + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + throw new UnsupportedOperationException("User Principal Lookup Service not supported"); + } + + @Override + public WatchService newWatchService() throws IOException { + throw new UnsupportedOperationException("Watch Service not supported"); + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystemProvider.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystemProvider.java new file mode 100644 index 0000000000..d8e0008f06 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraFileSystemProvider.java @@ -0,0 +1,724 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.fs; + +import io.seqera.http.HxClient; +import io.seqera.tower.plugin.TowerConfig; +import io.seqera.tower.plugin.TowerHxClientFactory; +import io.seqera.tower.plugin.datalink.DataLink; +import io.seqera.tower.plugin.datalink.DataLinkUtils; +import nextflow.SysEnv; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.nio.ByteBuffer; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.FileTime; +import java.nio.file.spi.FileSystemProvider; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * File System Provider for Seqera Platform Data-Link Paths + * + * @author Jorge Ejarque + */ +public class SeqeraFileSystemProvider extends FileSystemProvider { + + private static final Logger log = LoggerFactory.getLogger(SeqeraFileSystemProvider.class); + public static final String SCHEME = "seqera"; + + private final Map fileSystems = new HashMap<>(); + + @Override + public String getScheme() { + return SCHEME; + } + + protected SeqeraPath toSeqeraPath(Path path) { + if (!(path instanceof SeqeraPath)) { + throw new ProviderMismatchException(); + } + return (SeqeraPath) path; + } + + private void checkScheme(URI uri) { + final String scheme = uri.getScheme().toLowerCase(); + if (!scheme.equals(getScheme())) { + throw new IllegalArgumentException("Not a valid " + getScheme().toUpperCase() + " scheme: " + scheme); + } + } + + @Override + public synchronized FileSystem newFileSystem(URI uri, Map config) throws IOException { + checkScheme(uri); + + // URI format: seqera://datalink-name/optional/path + final String dataLinkName = uri.getHost(); + if (dataLinkName == null) { + throw new IllegalArgumentException("Data-link name must be specified in URI host"); + } + + final String key = buildFileSystemKey(uri, config); + if (fileSystems.containsKey(key)) { + throw new FileSystemAlreadyExistsException("File system already exists for: " + key); + } + + // Extract configuration + final TowerConfig twConfig = new TowerConfig(config, SysEnv.get()); + + if (twConfig.getAccessToken() == null) { + throw new IllegalArgumentException("Access token must be provided via 'token' config or TOWER_ACCESS_TOKEN environment variable"); + } + // Create HTTP client + final HxClient httpClient = TowerHxClientFactory.httpClient(twConfig.getAccessToken(), SysEnv.get("TOWER_REFRESH_TOKEN"),twConfig.getEndpoint(), twConfig.getRetryPolicy() ); + + final DataLink dataLink = DataLinkUtils.findDataLink(httpClient, twConfig.getEndpoint(), twConfig.getWorkspaceId(), dataLinkName); + + final SeqeraFileSystem fs = new SeqeraFileSystem(this, dataLink, twConfig.getEndpoint(), twConfig.getWorkspaceId(), httpClient); + fileSystems.put(key, fs); + return fs; + } + + private String buildFileSystemKey(URI uri, Map config) { + final String dataLinkName = uri.getHost(); + final String endpoint = config.containsKey("endpoint") + ? (String) config.get("endpoint") + : (System.getenv("TOWER_API_ENDPOINT") != null ? System.getenv("TOWER_API_ENDPOINT") : "https://api.cloud.seqera.io"); + final String workspaceId = config.containsKey("workspaceId") + ? (String) config.get("workspaceId") + : (System.getenv("TOWER_WORKSPACE_ID") != null ? System.getenv("TOWER_WORKSPACE_ID") : ""); + return endpoint + ":" + workspaceId + ":" + dataLinkName; + } + + @Override + public synchronized FileSystem getFileSystem(URI uri) throws FileSystemNotFoundException { + checkScheme(uri); + final String key = buildFileSystemKey(uri, new HashMap<>()); + final SeqeraFileSystem fs = fileSystems.get(key); + if (fs == null) { + throw new FileSystemNotFoundException("File system not found for: " + uri); + } + return fs; + } + + public synchronized FileSystem getFileSystemOrCreate(URI uri, Map config) throws IOException { + checkScheme(uri); + final String key = buildFileSystemKey(uri, config); + if (!fileSystems.containsKey(key)) { + return newFileSystem(uri, config); + } + return fileSystems.get(key); + } + + public synchronized FileSystem getFileSystemOrCreate(URI uri) throws IOException { + return getFileSystemOrCreate(uri, new HashMap<>()); + } + + @Override + public SeqeraPath getPath(URI uri) { + try { + final SeqeraFileSystem fs = (SeqeraFileSystem) getFileSystemOrCreate(uri); + return (SeqeraPath) fs.getPath(uri); + } catch (IOException e) { + throw new RuntimeException("Failed to get path for URI: " + uri, e); + } + } + + @Override + public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException { + final SeqeraPath seqPath = toSeqeraPath(path); + final SeqeraFileSystem fs = (SeqeraFileSystem) seqPath.getFileSystem(); + + // Check if write is requested + boolean write = false; + for (OpenOption opt : options) { + if (opt == StandardOpenOption.WRITE || opt == StandardOpenOption.CREATE || + opt == StandardOpenOption.CREATE_NEW || opt == StandardOpenOption.TRUNCATE_EXISTING) { + write = true; + break; + } + } + + if (!write) { + throw new IllegalArgumentException("Write option must be specified"); + } + + // Create a temporary file and return an OutputStream that will upload on close + final Path tempFile = Files.createTempFile("seqera-upload-", ".tmp"); + return new SeqeraOutputStream(tempFile, seqPath, fs); + } + + private static class SeqeraOutputStream extends OutputStream { + private final OutputStream delegate; + private final Path tempFile; + private final SeqeraPath targetPath; + private final SeqeraFileSystem fs; + + SeqeraOutputStream(Path tempFile, SeqeraPath targetPath, SeqeraFileSystem fs) throws IOException { + this.tempFile = tempFile; + this.targetPath = targetPath; + this.fs = fs; + this.delegate = Files.newOutputStream(tempFile); + } + + @Override + public void write(int b) throws IOException { + delegate.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + delegate.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + delegate.write(b, off, len); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + public void close() throws IOException { + delegate.close(); + try { + // Upload the file to Seqera Platform + DataLinkUtils.uploadFile( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + targetPath.getPathForApi(), + tempFile, + fs.getWorkspaceId() + ); + } finally { + // Clean up temp file + Files.deleteIfExists(tempFile); + } + } + } + + @Override + public InputStream newInputStream(Path path, OpenOption... options) throws IOException { + final SeqeraPath seqPath = toSeqeraPath(path); + final SeqeraFileSystem fs = (SeqeraFileSystem) seqPath.getFileSystem(); + + // Download to temp file and return input stream + final Path tempFile = Files.createTempFile("seqera-download-", ".tmp"); + try { + DataLinkUtils.downloadFile( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + seqPath.getPathForApi(), + tempFile, + fs.getWorkspaceId() + ); + return new DeleteOnCloseInputStream(tempFile); + } catch (Exception e) { + Files.deleteIfExists(tempFile); + throw e; + } + } + + private static class DeleteOnCloseInputStream extends InputStream { + private final InputStream delegate; + private final Path tempFile; + + DeleteOnCloseInputStream(Path tempFile) throws IOException { + this.tempFile = tempFile; + this.delegate = Files.newInputStream(tempFile); + } + + @Override + public int read() throws IOException { + return delegate.read(); + } + + @Override + public int read(byte[] b) throws IOException { + return delegate.read(b); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + return delegate.read(b, off, len); + } + + @Override + public void close() throws IOException { + try { + delegate.close(); + } finally { + Files.deleteIfExists(tempFile); + } + } + } + + @Override + public SeekableByteChannel newByteChannel(Path path, Set options, FileAttribute... attrs) throws IOException { + final SeqeraPath seqPath = toSeqeraPath(path); + + // Check if this is a read or write operation + boolean write = options.contains(StandardOpenOption.WRITE) || + options.contains(StandardOpenOption.CREATE) || + options.contains(StandardOpenOption.CREATE_NEW); + + if (write) { + throw new UnsupportedOperationException("SeekableByteChannel write not supported - use newOutputStream instead"); + } + + // For read, create a temp file and return a channel + final SeqeraFileSystem fs = (SeqeraFileSystem) seqPath.getFileSystem(); + final Path tempFile = Files.createTempFile("seqera-channel-", ".tmp"); + + try { + DataLinkUtils.downloadFile( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + seqPath.getPathForApi(), + tempFile, + fs.getWorkspaceId() + ); + return new DeleteOnCloseSeekableByteChannel(tempFile); + } catch (Exception e) { + Files.deleteIfExists(tempFile); + throw e; + } + } + + private static class DeleteOnCloseSeekableByteChannel implements SeekableByteChannel { + private final SeekableByteChannel delegate; + private final Path tempFile; + + DeleteOnCloseSeekableByteChannel(Path tempFile) throws IOException { + this.tempFile = tempFile; + this.delegate = Files.newByteChannel(tempFile); + } + + @Override + public int read(ByteBuffer dst) throws IOException { + return delegate.read(dst); + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public long position() throws IOException { + return delegate.position(); + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + delegate.position(newPosition); + return this; + } + + @Override + public long size() throws IOException { + return delegate.size(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public boolean isOpen() { + return delegate.isOpen(); + } + + @Override + public void close() throws IOException { + try { + delegate.close(); + } finally { + Files.deleteIfExists(tempFile); + } + } + } + + @Override + public DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) throws IOException { + final SeqeraPath seqPath = toSeqeraPath(dir); + final SeqeraFileSystem fs = (SeqeraFileSystem) seqPath.getFileSystem(); + + // Use browse API to list files + final List files = listFiles(fs, seqPath.getPathForApi()); + + return new DirectoryStream() { + @Override + public Iterator iterator() { + return files.iterator(); + } + + @Override + public void close() throws IOException { + // Nothing to close + } + }; + } + + private List listFiles(SeqeraFileSystem fs, String path) throws IOException { + // Use utility method to get file names + final List fileNames = DataLinkUtils.listFiles( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + path, + fs.getWorkspaceId() + ); + + // Convert file names to Path objects + final List result = new ArrayList<>(); + for (String name : fileNames) { + final String fullPath = path != null && !path.isEmpty() ? path + "/" + name : name; + result.add(new SeqeraPath(fs, fullPath)); + } + return result; + } + + @Override + public void createDirectory(Path dir, FileAttribute... attrs) throws IOException { + // Data-links don't require explicit directory creation + log.debug("Directory creation not required for Seqera paths: {}", dir); + } + + @Override + public void delete(Path path) throws IOException { + final SeqeraPath seqPath = toSeqeraPath(path); + final SeqeraFileSystem fs = (SeqeraFileSystem) seqPath.getFileSystem(); + + // Use utility method to delete file + DataLinkUtils.deleteFile( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + seqPath.getPathForApi(), + fs.getWorkspaceId() + ); + } + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + // Determine if source and target are Seqera paths or local paths + final boolean sourceIsSeqera = source instanceof SeqeraPath; + final boolean targetIsSeqera = target instanceof SeqeraPath; + + if (!sourceIsSeqera && !targetIsSeqera) { + // Both are local paths - not our responsibility + throw new UnsupportedOperationException("Copy between local paths should use Files.copy()"); + } + + if (sourceIsSeqera && targetIsSeqera) { + // Both are Seqera paths - server-side copy not supported yet + throw new UnsupportedOperationException("Copy between Seqera paths not supported - download and re-upload instead"); + } + + // Handle copy from local to Seqera or Seqera to local + if (sourceIsSeqera) { + // Download from Seqera to local + copyFromSeqeraToLocal((SeqeraPath) source, target); + } else { + // Upload from local to Seqera + copyFromLocalToSeqera(source, (SeqeraPath) target); + } + } + + private void copyFromSeqeraToLocal(SeqeraPath source, Path target) throws IOException { + final SeqeraFileSystem fs = (SeqeraFileSystem) source.getFileSystem(); + + // Check if source is a directory or file + final BasicFileAttributes attrs = readAttributes(source, BasicFileAttributes.class); + + if (attrs.isDirectory()) { + // Download folder recursively + DataLinkUtils.downloadFolder( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + source.getPathForApi(), + target, + fs.getWorkspaceId() + ); + } else { + // Download single file + DataLinkUtils.downloadFile( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + source.getPathForApi(), + target, + fs.getWorkspaceId() + ); + } + } + + private void copyFromLocalToSeqera(Path source, SeqeraPath target) throws IOException { + final SeqeraFileSystem fs = (SeqeraFileSystem) target.getFileSystem(); + + if (Files.isDirectory(source)) { + // Upload folder recursively + DataLinkUtils.uploadFolder( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + target.getPathForApi(), + source, + fs.getWorkspaceId() + ); + } else { + // Upload single file + DataLinkUtils.uploadFile( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + target.getPathForApi(), + source, + fs.getWorkspaceId() + ); + } + } + + @Override + public void move(Path source, Path target, CopyOption... options) throws IOException { + throw new UnsupportedOperationException("Move not supported"); + } + + @Override + public boolean isSameFile(Path path, Path path2) throws IOException { + return path.equals(path2); + } + + @Override + public boolean isHidden(Path path) throws IOException { + return false; + } + + @Override + public FileStore getFileStore(Path path) throws IOException { + throw new UnsupportedOperationException("File store not supported"); + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + final SeqeraPath seqPath = toSeqeraPath(path); + final SeqeraFileSystem fs = (SeqeraFileSystem) seqPath.getFileSystem(); + + for (AccessMode m : modes) { + if (m == AccessMode.EXECUTE) { + throw new AccessDeniedException("Execute mode not supported"); + } + } + + // Check if the file exists using getFileDetails + final DataLinkUtils.DataLinkItem item = DataLinkUtils.getFileDetails( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + seqPath.getPathForApi(), + fs.getWorkspaceId() + ); + + if (item == null) { + throw new NoSuchFileException(path.toString()); + } + } + + @Override + @SuppressWarnings("unchecked") + public V getFileAttributeView(Path path, Class type, LinkOption... options) { + if (type == BasicFileAttributeView.class) { + return (V) new SeqeraFileAttributeView(toSeqeraPath(path)); + } + return null; + } + + @Override + @SuppressWarnings("unchecked") + public A readAttributes(Path path, Class type, LinkOption... options) throws IOException { + if (type == BasicFileAttributes.class || BasicFileAttributes.class.isAssignableFrom(type)) { + return (A) new SeqeraFileAttributes(toSeqeraPath(path)); + } + throw new UnsupportedOperationException("Attributes type not supported: " + type); + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) throws IOException { + throw new UnsupportedOperationException("Read file attributes not supported"); + } + + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) throws IOException { + throw new UnsupportedOperationException("Set file attributes not supported"); + } + + /** + * Check if a file attribute view is supported + */ + public boolean supportsFileAttributeView(String name) { + return "basic".equals(name); + } + + private static class SeqeraFileAttributeView implements BasicFileAttributeView { + private final SeqeraPath path; + + SeqeraFileAttributeView(SeqeraPath path) { + this.path = path; + } + + @Override + public String name() { + return "basic"; + } + + @Override + public BasicFileAttributes readAttributes() throws IOException { + return new SeqeraFileAttributes(path); + } + + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException { + throw new UnsupportedOperationException("Set times not supported"); + } + } + + private static class SeqeraFileAttributes implements BasicFileAttributes { + private final SeqeraPath path; + private final FileTime creationTime; + private final FileTime lastModifiedTime; + private final boolean isDirectory; + private final long size; + + SeqeraFileAttributes(SeqeraPath path) throws IOException { + this.path = path; + this.creationTime = FileTime.from(Instant.now()); + this.lastModifiedTime = FileTime.from(Instant.now()); + + final String pathStr = path.getPathForApi(); + + // If path is empty, it's the root which is a directory + if (pathStr == null || pathStr.isEmpty()) { + this.isDirectory = true; + this.size = 0L; + } else { + // Fetch file details from the API + final SeqeraFileSystem fs = (SeqeraFileSystem) path.getFileSystem(); + final DataLinkUtils.DataLinkItem item = DataLinkUtils.getFileDetails( + fs.getHttpClient(), + fs.getEndpoint(), + fs.getDataLink(), + pathStr, + fs.getWorkspaceId() + ); + + if (item == null) { + throw new NoSuchFileException(path.toString()); + } + + this.isDirectory = item.getType() == DataLinkUtils.DataLinkItemType.FOLDER; + this.size = item.getSize() != null ? item.getSize().longValue() : 0L; + } + } + + @Override + public FileTime lastModifiedTime() { + return lastModifiedTime; + } + + @Override + public FileTime lastAccessTime() { + return lastModifiedTime; + } + + @Override + public FileTime creationTime() { + return creationTime; + } + + @Override + public boolean isRegularFile() { + return !isDirectory; + } + + @Override + public boolean isDirectory() { + return isDirectory; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public long size() { + return size; + } + + @Override + public Object fileKey() { + return null; + } + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPath.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPath.java new file mode 100644 index 0000000000..22497d8e90 --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPath.java @@ -0,0 +1,366 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.fs; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Iterator; +import java.util.Objects; + +/** + * Seqera Platform Data-Link file system path + * + * @author Jorge Ejarque + */ +public class SeqeraPath implements Path { + + private static final Logger log = LoggerFactory.getLogger(SeqeraPath.class); + + public static final String SEPARATOR = "/"; + public static final String SCHEME = "seqera"; + public static final String SEQERA_PROT = SCHEME + "://"; + + private static final String[] EMPTY = new String[]{}; + + private final SeqeraFileSystem fileSystem; + private final String path; + + /* + * Only needed to prevent serialization issues - see https://github.com/nextflow-io/nextflow/issues/5208 + */ + protected SeqeraPath() { + this.fileSystem = null; + this.path = null; + } + + public SeqeraPath(SeqeraFileSystem fs, URI uri) { + if (!SCHEME.equals(uri.getScheme())) { + throw new IllegalArgumentException("Invalid Seqera URI - scheme must be '" + SCHEME + "'"); + } + this.fileSystem = fs; + // URI format: seqera://datalink-name/path/to/file + this.path = normalizePath(uri.getPath() != null ? uri.getPath() : ""); + } + + public SeqeraPath(SeqeraFileSystem fs, String path) { + this.fileSystem = fs; + this.path = normalizePath(path); + } + + private static String normalizePath(String path) { + if (path == null || path.isEmpty() || path.equals(SEPARATOR)) { + return ""; + } + // Normalize and remove leading/trailing separators + path = path.trim(); + if (path.startsWith(SEPARATOR)) { + path = path.substring(1); + } + if (path.endsWith(SEPARATOR) && path.length() > 1) { + path = path.substring(0, path.length() - 1); + } + return path; + } + + public static boolean isSeqeraUri(String path) { + return path != null && path.startsWith(SEQERA_PROT); + } + + public static URI asUri(String path) { + if (path == null) { + throw new IllegalArgumentException("Missing 'path' argument"); + } + if (!path.startsWith(SEQERA_PROT)) { + throw new IllegalArgumentException("Invalid Seqera file system path URI - it must start with '" + SEQERA_PROT + "' prefix"); + } + return URI.create(path); + } + + @Override + public FileSystem getFileSystem() { + return fileSystem; + } + + @Override + public boolean isAbsolute() { + return fileSystem != null; + } + + @Override + public Path getRoot() { + return new SeqeraPath(fileSystem, ""); + } + + @Override + public Path getFileName() { + if (path == null || path.isEmpty()) { + return null; + } + final int idx = path.lastIndexOf(SEPARATOR); + return idx == -1 + ? new SeqeraPath(null, path) + : new SeqeraPath(null, path.substring(idx + 1)); + } + + @Override + public Path getParent() { + if (path == null || path.isEmpty()) { + return null; + } + final int idx = path.lastIndexOf(SEPARATOR); + if (idx == -1) { + return new SeqeraPath(fileSystem, ""); + } + return new SeqeraPath(fileSystem, path.substring(0, idx)); + } + + @Override + public int getNameCount() { + if (path == null || path.isEmpty()) { + return 0; + } + return path.split(SEPARATOR).length; + } + + @Override + public Path getName(int index) { + if (index < 0) { + throw new IllegalArgumentException("Path name index cannot be less than zero - offending value: " + index); + } + final String[] parts = path.split(SEPARATOR); + if (index >= parts.length) { + throw new IllegalArgumentException("Index out of bounds: " + index); + } + return new SeqeraPath(null, parts[index]); + } + + @Override + public Path subpath(int beginIndex, int endIndex) { + if (beginIndex < 0) { + throw new IllegalArgumentException("subpath begin index cannot be less than zero - offending value: " + beginIndex); + } + if (beginIndex >= endIndex) { + throw new IllegalArgumentException("begin index must be less than end index"); + } + final String[] parts = path.split(SEPARATOR); + if (endIndex > parts.length) { + throw new IllegalArgumentException("end index out of bounds: " + endIndex); + } + final StringBuilder sb = new StringBuilder(); + for (int i = beginIndex; i < endIndex; i++) { + if (i > beginIndex) { + sb.append(SEPARATOR); + } + sb.append(parts[i]); + } + return new SeqeraPath(beginIndex == 0 ? fileSystem : null, sb.toString()); + } + + @Override + public Path normalize() { + return new SeqeraPath(fileSystem, normalizePath(path)); + } + + @Override + public Path resolve(Path other) { + if (!(other instanceof SeqeraPath)) { + throw new ProviderMismatchException(); + } + + final SeqeraPath that = (SeqeraPath) other; + + if (that.fileSystem != null && this.fileSystem != that.fileSystem) { + return other; + } + if (that.isAbsolute()) { + return that; + } + if (this.path == null || this.path.isEmpty()) { + return that; + } + return new SeqeraPath(fileSystem, path + SEPARATOR + that.path); + } + + @Override + public Path resolve(String other) { + if (other == null || other.isEmpty()) { + return this; + } + // If it's a Seqera URI, parse it + if (other.startsWith(SEQERA_PROT)) { + final Path that = fileSystem.provider().getPath(asUri(other)); + return resolve(that); + } + // Otherwise, treat as relative path + return resolve(new SeqeraPath(null, other)); + } + + @Override + public Path relativize(Path other) { + if (!(other instanceof SeqeraPath)) { + throw new ProviderMismatchException(); + } + final SeqeraPath that = (SeqeraPath) other; + if (this.isAbsolute() != that.isAbsolute()) { + throw new IllegalArgumentException("Cannot compare absolute with relative paths"); + } + + final String[] thisParts = this.path != null && !this.path.isEmpty() ? this.path.split(SEPARATOR) : new String[0]; + final String[] thatParts = that.path != null && !that.path.isEmpty() ? that.path.split(SEPARATOR) : new String[0]; + + // Find common prefix + int common = 0; + while (common < thisParts.length && common < thatParts.length && thisParts[common].equals(thatParts[common])) { + common++; + } + + // Build relative path + final StringBuilder result = new StringBuilder(); + final int upLevels = thisParts.length - common; + for (int i = 0; i < upLevels; i++) { + if (i > 0) { + result.append(SEPARATOR); + } + result.append(".."); + } + for (int i = common; i < thatParts.length; i++) { + if (result.length() > 0) { + result.append(SEPARATOR); + } + result.append(thatParts[i]); + } + + return new SeqeraPath(null, result.toString()); + } + + @Override + public URI toUri() { + final String dataLinkName = fileSystem != null ? fileSystem.getDataLinkName() : "unknown"; + final String uriPath = path != null && !path.isEmpty() ? SEPARATOR + path : ""; + return URI.create(SEQERA_PROT + dataLinkName + uriPath); + } + + public String toUriString() { + return toUri().toString(); + } + + @Override + public Path toAbsolutePath() { + return this; + } + + @Override + public Path toRealPath(LinkOption... options) throws IOException { + // For Seqera paths, real path is the same as the path itself + return this; + } + + @Override + public File toFile() { + throw new UnsupportedOperationException("toFile not supported by SeqeraPath"); + } + + @Override + public WatchKey register(WatchService watcher, WatchEvent.Kind[] events, WatchEvent.Modifier... modifiers) throws IOException { + throw new UnsupportedOperationException("Register not supported by SeqeraPath"); + } + + @Override + public WatchKey register(WatchService watcher, WatchEvent.Kind... events) throws IOException { + throw new UnsupportedOperationException("Register not supported by SeqeraPath"); + } + + @Override + public Iterator iterator() { + return new Iterator() { + private int index = 0; + private final String[] parts = path != null && !path.isEmpty() ? path.split(SEPARATOR) : new String[0]; + + @Override + public boolean hasNext() { + return index < parts.length; + } + + @Override + public Path next() { + return new SeqeraPath(null, parts[index++]); + } + }; + } + + @Override + public int compareTo(Path other) { + return toString().compareTo(other.toString()); + } + + @Override + public boolean startsWith(Path other) { + return startsWith(other.toString()); + } + + @Override + public boolean startsWith(String other) { + return path != null && path.startsWith(normalizePath(other)); + } + + @Override + public boolean endsWith(Path other) { + return endsWith(other.toString()); + } + + @Override + public boolean endsWith(String other) { + return path != null && path.endsWith(normalizePath(other)); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof SeqeraPath)) { + return false; + } + final SeqeraPath that = (SeqeraPath) other; + return Objects.equals(this.fileSystem, that.fileSystem) && Objects.equals(this.path, that.path); + } + + @Override + public int hashCode() { + return Objects.hash(fileSystem, path); + } + + @Override + public String toString() { + return path != null ? path : ""; + } + + /** + * Get the path string for API calls + */ + public String getPathForApi() { + return path != null ? path : ""; + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPathFactory.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPathFactory.java new file mode 100644 index 0000000000..f5aa55fb0f --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/fs/SeqeraPathFactory.java @@ -0,0 +1,68 @@ +/* + * Copyright 2013-2025, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.seqera.tower.plugin.fs; + +import nextflow.file.FileSystemPathFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.file.Path; + +/** + * Factory for creating Seqera Platform Data-Link file system paths + * + * @author Jorge Ejarque + */ +public class SeqeraPathFactory extends FileSystemPathFactory { + + private static final Logger log = LoggerFactory.getLogger(SeqeraPathFactory.class); + + @Override + protected Path parseUri(String uri) { + if (!uri.startsWith(SeqeraPath.SEQERA_PROT)) { + return null; + } + + try { + final SeqeraFileSystemProvider provider = new SeqeraFileSystemProvider(); + final java.net.URI parsedUri = SeqeraPath.asUri(uri); + return provider.getPath(parsedUri); + } catch (Exception e) { + log.warn("Failed to parse Seqera URI: {}", uri, e); + return null; + } + } + + @Override + protected String toUriString(Path path) { + if (path instanceof SeqeraPath) { + return ((SeqeraPath) path).toUriString(); + } + return null; + } + + @Override + protected String getBashLib(Path path) { + return null; + } + + @Override + protected String getUploadCmd(String source, Path target) { + // Could implement a custom upload command here + return null; + } +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy index 26afa58734..df18470cee 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy @@ -23,6 +23,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.tower.plugin.BaseCommandImpl import io.seqera.tower.plugin.TowerClient +import io.seqera.tower.plugin.datalink.DataLinkUtils import nextflow.BuildInfo import nextflow.cli.CmdLaunch import nextflow.util.ColorUtil @@ -93,13 +94,12 @@ class LaunchCommandImpl extends BaseCommandImpl implements CmdLaunch.LaunchComma @Override void launch(CmdLaunch.LaunchOptions options) { printBanner(options) - - // Validate and resolve pipeline - final resolvedPipelineUrl = validateAndResolvePipeline(options.pipeline) - // Initialize launch context with auth and workspace info final context = initializeLaunchContext(options) + // Validate and resolve pipeline + final resolvedPipelineUrl = validateAndResolvePipeline(options, context) + // Build and submit the launch request final result = submitWorkflowLaunch(options, context, resolvedPipelineUrl) @@ -119,20 +119,49 @@ class LaunchCommandImpl extends BaseCommandImpl implements CmdLaunch.LaunchComma // ===== Launch Phases ===== /** - * Validate pipeline path and resolve to full repository URL + * Validate pipeline path and resolve pipeline. + * If the pipeline is a remote URL, just resolve the URL. + * If the pipeline is a local path it manages it as a local git folder, pushed changes and resolve url and revision. */ - private String validateAndResolvePipeline(String pipeline) { - log.debug "Pipeline repository: ${pipeline}" + private String validateAndResolvePipeline(CmdLaunch.LaunchOptions options, LaunchContext context) { + log.debug "Pipeline repository: ${options.pipeline}" - if (isLocalPath(pipeline)) { - log.debug "Rejecting local file path: ${pipeline}" - throw new AbortOperationException("Local file paths are not supported. Please provide a remote repository URL.") + if (isLocalPath(options.pipeline)) { + return manageLocalPipeline(options, context) } - final resolvedUrl = resolvePipelineUrl(pipeline) + final resolvedUrl = resolvePipelineUrl(options.pipeline) log.debug "Resolved pipeline URL: ${resolvedUrl}" return resolvedUrl } + /** + * Resolve repo URL and branch associated to the local folder. + * Initialize as git folder, if it is not yet initialized, and pushes it as a 'seqera-git-remote' + * to the working dir specified in the launch context. + * + * @param options Launch Options where folder name and requested revision are specified + * @param launchContext Launch context to find + * @return the resolved repo URL associated to the folder + */ + private String manageLocalPipeline(CmdLaunch.LaunchOptions options, LaunchContext launchContext) { + File dir = new File(options.pipeline) + if (!dir.exists()) + throw new AbortOperationException("Directory $dir doesn't exist") + final manager = new PushManager(dir) + String resolvedUrl = manager.isLocalGit() ? manager.resolveRepository() : resolveRepositoryFormContext(launchContext) + String pushedBranch = manager.push(resolvedUrl, options.revision) + log.debug "Resolved pipeline URL: ${resolvedUrl} (revision: ${pushedBranch})" + options.revision = pushedBranch + return resolvedUrl + } + + private String resolveRepositoryFormContext(LaunchContext launchContext) { + final uri = URI.create(launchContext.workDir) + final datalink = DataLinkUtils.findDataLink(createHttpClient(launchContext.accessToken), launchContext.apiEndpoint, launchContext.workspaceId?.toString(), uri.getHost()) + if (datalink.provider.toLowerCase() != 'seqeracompute') + throw new AbortOperationException("Data link type associated to '${launchContext.workDir}' is not 'seqeracompute' - Local pipelines only available for Seqera Compute environments") + return new URI('seqera', uri.host, "${uri.path}/launch-repos/nf-launch-${UUID.randomUUID().toString()}" , uri.fragment).toString() + } /** * Initialize launch context by loading config and resolving workspace/compute environment @@ -921,6 +950,9 @@ class LaunchCommandImpl extends BaseCommandImpl implements CmdLaunch.LaunchComma if (path.contains('\\') || path ==~ /^[A-Za-z]:.*/) { return true } + if (!AssetManager.isUrl(path) && new File(path).exists()) { + return true + } return false } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy new file mode 100644 index 0000000000..6524cde8ca --- /dev/null +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy @@ -0,0 +1,416 @@ +package io.seqera.tower.plugin.launch + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.exception.AbortOperationException +import nextflow.scm.AssetManager +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.api.Status +import org.eclipse.jgit.transport.RemoteConfig +import org.eclipse.jgit.transport.URIish + + +@Slf4j +@CompileStatic +class PushManager { + private static final String DEFAULT_BRANCH = 'main' + File folder + + PushManager(File folder){ + this.folder = folder + } + + boolean isLocalGit(){ + final gitDir = new File(folder, '.git') + return gitDir.exists() + } + + String push(String repo, String requestedBranch){ + def remoteName = "origin" + def isNewRepo = false + def revision = DEFAULT_BRANCH + if( isLocalGit() ) { + log.debug "Found existing git repository in ${folder.absolutePath}" + remoteName = validateExistingRepo(repo) + def currentBranch = getCurrentBranch() + if( requestedBranch && currentBranch && currentBranch != requestedBranch ) { + throw new AbortOperationException( + "Current branch '${currentBranch}' does not match requested branch '${requestedBranch}'.\n" + + "Please checkout to branch '${requestedBranch}' before pushing or specify the correct branch with -r option." + ) + } else if( !requestedBranch && currentBranch ){ + revision = currentBranch + } + } else { + log.debug "No git repository found, initializing new one" + revision = requestedBranch ?: revision + initializeRepo(repo) + isNewRepo = true + } + + checkFileSizes() + manageNextflowGitignore() + stageAndCommitFiles() + def manager = new AssetManager(folder, repo) + manager.upload(revision, remoteName, isNewRepo) + log.info "Successfully pushed to ${repo} (revision: ${revision})" + return revision + } + + private String validateExistingRepo(String expectedRepo) { + def git = Git.open(folder) + + try { + def remotes = git.remoteList().call() + + // Find all remotes and check if any matches the expected repo + def matchingRemote = null + + for( RemoteConfig remote : remotes ) { + if( remote.URIs ) { + def remoteUrl = remote.URIs[0].toString() + def normalizedRemote = normalizeRepoUrl(remoteUrl) + def normalizedExpected = normalizeRepoUrl(expectedRepo) + + if( normalizedRemote == normalizedExpected ) { + matchingRemote = remote.name + break + } + } + } + + if( !matchingRemote ) { + def remotesList = remotes.collect { remote -> + def url = remote.URIs ? remote.URIs[0].toString() : 'no URL' + " ${remote.name}: ${url}" + }.join('\n') + + throw new AbortOperationException( + "Repository URL not found in remotes!\n" + + " Expected repository: ${expectedRepo}\n" + + " Available remotes:\n${remotesList}\n" + + "Please add the repository as a remote or specify the correct repository." + ) + } + + return matchingRemote + } + finally { + git.close() + } + } + + private String normalizeRepoUrl(String url) { + return url?.toLowerCase()?.replaceAll(/\.git$/, '')?.replaceAll(/\/$/, '') + } + + private String getCurrentBranch() { + def git = Git.open(folder) + + try { + def head = git.getRepository().findRef("HEAD") + if( !head ) { + log.debug "No HEAD found, assuming new repository. Returning default" + return null + } + + if( !head.isSymbolic() ) { + log.debug "HEAD is not symbolic (detached state)" + throw new AbortOperationException("Repository is in detached HEAD state. Please checkout to a branch before pushing.") + } + return git.getRepository().getBranch() + } finally { + git.close() + } + } + + private void initializeRepo(String repo) { + log.debug "Initializing git repository in ${folder.absolutePath}" + def git = Git.init().setDirectory(folder).call() + + // Add remote origin + git.remoteAdd() + .setName("origin") + .setUri(new URIish(repo)) + .call() + + git.close() + } + + private void checkFileSizes(int maxSizeMB = 10) { + def maxSizeBytes = maxSizeMB * 1024 * 1024 + def git = Git.open(folder) + + try { + // Get Git status to find files that would be committed + def status = git.status().call() + def filesToBeCommitted = [] + + // Add untracked files + filesToBeCommitted.addAll(status.untracked) + // Add modified files + filesToBeCommitted.addAll(status.modified) + // Add added files + filesToBeCommitted.addAll(status.added) + + def largeFiles = [] + + filesToBeCommitted.each { relativePath -> + def file = new File(folder, relativePath as String) + if( file.exists() && file.isFile() && file.length() > maxSizeBytes ) { + def fileEntry = [ + file: file, + relativePath: relativePath, + sizeMB: file.length() / (1024 * 1024) + ] + largeFiles.add(fileEntry) + } + } + + if( largeFiles ) { + log.warn "Found ${largeFiles.size()} large files that would be committed:" + largeFiles.each { entry -> + def sizeMB = entry['sizeMB'] as Double + log.warn " ${entry['relativePath']}: ${String.format('%.1f', sizeMB)} MB" + } + + print "Do you want to push these large files? [y/N]: " + def response = System.in.newReader().readLine()?.trim()?.toLowerCase() + + if( response != 'y' && response != 'yes' ) { + // Add large files to .gitignore + def relativePaths = largeFiles.collect { entry -> entry['relativePath'] as String } + addToGitignore(relativePaths) + println "Files have been added to .gitignore" + } + } + } + finally { + git.close() + } + } + + private void addToGitignore(List filenames) { + def gitignoreFile = new File(folder, '.gitignore') + def content = [] + + if( gitignoreFile.exists() ) { + content = gitignoreFile.readLines() + } + + filenames.each { filename -> + if( !content.contains(filename) ) { + content.add(filename) + } + } + + gitignoreFile.text = content.join('\n') + '\n' + log.info "Added ${filenames.size()} large files to .gitignore" + } + + private void manageNextflowGitignore() { + def gitignoreFile = new File(folder, '.gitignore') + List content = [] + + if( gitignoreFile.exists() ) { + content = gitignoreFile.readLines() + } + + // Default Nextflow entries to add + def nextflowEntries = [ + '.nextflow', + '.nextflow.log*' + ] + + def added = [] + nextflowEntries.each { entry -> + if( !content.contains(entry) ) { + content.add(entry) + added.add(entry) + } + } + + // Check for work directory + def workDirs = findWorkDirectories() + if( workDirs ) { + def workEntriesToAdd = promptForWorkDirectories(workDirs, content) + workEntriesToAdd.each { workDir -> + if( !content.contains(workDir) ) { + content.add(workDir) + added.add(workDir) + } + } + } + + if( added ) { + gitignoreFile.text = content.join('\n') + '\n' + log.info "Added ${added.size()} Nextflow entries to .gitignore: ${added.join(', ')}" + } else { + log.debug "All Nextflow entries already present in .gitignore" + } + } + + private List findWorkDirectories() { + List workDirs = [] + + // Check for the default Nextflow work directory + def workDir = new File(folder, 'work') + if( workDir.exists() && workDir.isDirectory() ) { + workDirs.add('work') + } + + return workDirs + } + + private List promptForWorkDirectories(List workDirs, List currentGitignore) { + List toAdd = [] + + workDirs.each { workDir -> + // Check if already in .gitignore + if( currentGitignore.contains(workDir) ) { + log.debug "Work directory '${workDir}' already in .gitignore" + return // Skip this directory + } + + println "Found Nextflow work directory: ${workDir}" + print "Do you want to add '${workDir}' to .gitignore? [Y/n]: " + def response = System.in.newReader().readLine()?.trim()?.toLowerCase() + + // Default to 'yes' if empty response or 'y'/'yes' + if( !response || response == 'y' || response == 'yes' ) { + toAdd.add(workDir) + log.info "Will add '${workDir}' to .gitignore" + } else { + log.info "Skipping '${workDir}'" + } + } + + return toAdd + } + + private void stageAndCommitFiles(String message='Push from nextflow') { + def git = Git.open(folder) + + try { + // Add all files + git.add().addFilepattern(".").call() + + // Check if there are any changes to commit + def status = git.status().call() + if( status.clean ) { + log.info "No changes to commit" + return + } + + showAndConfirmStagedFiles(status, git) + + // Commit changes + git.commit() + .setMessage(message) + .call() + + log.debug "Committed changes with message: ${message}" + } + finally { + git.close() + } + } + + private void showAndConfirmStagedFiles(Status status, Git git) { + def stagedFiles = [] + stagedFiles.addAll(status.added) + stagedFiles.addAll(status.changed) + + if( stagedFiles ) { + println "\nFiles to be committed:" + stagedFiles.each { file -> + println " ${file}" + } + + print "\nDo you want to commit these files? [Y/n]: " + def response = System.in.newReader().readLine()?.trim()?.toLowerCase() + + // Default to 'yes' if empty response or 'y'/'yes' + if( response && response != 'y' && response != 'yes' ) { + log.info "Commit cancelled by user" + + // Unstage all files + git.reset().call() + log.info "Files have been unstaged" + + throw new AbortOperationException("Commit cancelled by user") + } + } + } + + String resolveRepository() { + def gitDir = new File(folder, '.git') + + if( !gitDir.exists() ) { + throw new AbortOperationException("No git repository found and no repository URL provided. Please specify a repository with -repo parameter.") + } + + def git = Git.open(folder) + + try { + def remotes = git.remoteList().call() + + if( remotes.empty ) { + throw new AbortOperationException("No remotes configured in git repository. Please add a remote or specify a repository with -repo parameter.") + } + + if( remotes.size() == 1 ) { + def remote = remotes[0] + def remoteUrl = remote.URIs[0].toString() + log.info "Using remote '${remote.name}': ${remoteUrl}" + return remoteUrl + } + + // Multiple remotes - ask user to choose + return selectRemoteFromUser(remotes) + } + finally { + git.close() + } + } + + private static String selectRemoteFromUser(List remotes) { + println "Multiple remotes found. Please select which remote to push to:" + + def remoteOptions = [:] + remotes.eachWithIndex { remote, index -> + def remoteUrl = remote.URIs[0].toString() + def remoteInfo = [name: remote.name, url: remoteUrl] + remoteOptions[index + 1] = remoteInfo + println " ${index + 1}. ${remote.name}: ${remoteUrl}" + } + + println " ${remotes.size() + 1}. Cancel" + + while( true ) { + print "Enter your choice [1-${remotes.size() + 1}]: " + def input = System.in.newReader().readLine()?.trim() + + try { + def choice = Integer.parseInt(input) + + if( choice == remotes.size() + 1 ) { + throw new AbortOperationException("Push operation cancelled by user.") + } + + if( choice >= 1 && choice <= remotes.size() ) { + def selected = remoteOptions[choice] + log.info "Selected remote '${selected['name']}': ${selected['url']}" + return selected['url'] + } + + println "Invalid choice. Please enter a number between 1 and ${remotes.size() + 1}." + } + catch( NumberFormatException ignored ) { + println "Invalid input. Please enter a number." + } + } + } + + +} diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy index 5678f1246a..c9030931ba 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraProviderConfig.groovy @@ -19,6 +19,8 @@ package io.seqera.tower.plugin.scm import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.tower.plugin.BaseCommandImpl +import io.seqera.tower.plugin.TowerConfig +import io.seqera.tower.plugin.TowerRetryPolicy import nextflow.Const import nextflow.Global import nextflow.SysEnv @@ -36,9 +38,7 @@ import nextflow.scm.ProviderConfig @CompileStatic class SeqeraProviderConfig extends ProviderConfig { - private String endpoint = 'https://api.cloud.seqera.io' - private String accessToken - private String workspaceId + private TowerConfig config SeqeraProviderConfig(String name, Map values) { super(name, [server: "seqera://$name"] + values) @@ -56,26 +56,27 @@ class SeqeraProviderConfig extends ProviderConfig { // Merge with SCM values - final config = towerConfig + values - endpoint = PlatformHelper.getEndpoint(config, SysEnv.get()) - accessToken = PlatformHelper.getAccessToken(config, SysEnv.get()) - workspaceId = PlatformHelper.getWorkspaceId(config, SysEnv.get()) + config = new TowerConfig(new HashMap(towerConfig + values), SysEnv.get()) - if (!accessToken) { + if (!config.accessToken) { throw new AbortOperationException("Seqera Platform access token not configured. Set TOWER_ACCESS_TOKEN environment variable or configure it in nextflow.config") } } String getEndpoint() { - this.endpoint + this.config.endpoint } String getAccessToken() { - this.accessToken + this.config.accessToken } String getWorkspaceId() { - this.workspaceId + this.config.workspaceId + } + + TowerRetryPolicy getRetryPolicy() { + this.config.retryPolicy } @Override diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy index 6d40bb05d5..592c597a01 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/SeqeraRepositoryProvider.groovy @@ -63,6 +63,9 @@ class SeqeraRepositoryProvider extends RepositoryProvider { if (providerConfig.workspaceId) { credentials.setWorkspaceId(providerConfig.workspaceId) } + if (providerConfig.retryPolicy){ + credentials.setRetryPolicy(providerConfig.retryPolicy) + } return credentials } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java index 40a21be6e3..6931c1fb95 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraBaseConnection.java @@ -16,34 +16,23 @@ package io.seqera.tower.plugin.scm.jgit; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import io.seqera.http.HxClient; +import io.seqera.tower.plugin.TowerHxClientFactory; +import io.seqera.tower.plugin.datalink.DataLink; +import nextflow.SysEnv; import org.eclipse.jgit.lib.*; import org.eclipse.jgit.transport.Connection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URLEncoder; import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; + +import static io.seqera.tower.plugin.datalink.DataLinkUtils.*; /** * Base class for connections with Seqera Platform data-links git-remote compatibility @@ -61,7 +50,6 @@ public class SeqeraBaseConnection implements Connection { protected String endpoint; protected String workspaceId; private final Map advertisedRefs = new HashMap(); - private final Gson gson = new Gson(); public SeqeraBaseConnection(TransportSeqera transport) { @@ -72,10 +60,10 @@ public SeqeraBaseConnection(TransportSeqera transport) { SeqeraGitCredentialsProvider credentials = (SeqeraGitCredentialsProvider) transport.getCredentialsProvider(); this.endpoint = credentials.getEndpoint(); this.workspaceId = credentials.getWorkspaceId(); - this.httpClient = createHttpClient(credentials); - this.dataLink = findDataLink(dataLinkName); + this.httpClient = TowerHxClientFactory.httpClient(credentials.getAccessToken(), SysEnv.get("TOWER_REFRESH_TOKEN"), credentials.getEndpoint(), credentials.getRetryPolicy()); + this.dataLink = findDataLink(this.httpClient, this.endpoint, this.workspaceId, dataLinkName); loadRefsMap(); - log.debug("Created Seqera Connection for dataLink={} path={}", dataLink.id, repoPath); + log.debug("Created Seqera Connection for dataLink={} path={}", dataLink.getId(), repoPath); } protected HxClient createHttpClient(SeqeraGitCredentialsProvider credentials) { @@ -111,334 +99,24 @@ protected HxClient createHttpClient(SeqeraGitCredentialsProvider credentials) { protected String getDefaultBranchRef() throws IOException { String path = repoPath + "/HEAD"; - byte[] content = downloadFile(path); + byte[] content = getFileContent(httpClient, endpoint, dataLink, path, workspaceId);; if (content != null) { return new String(content); } return null; } - /** - * Find DatalinkId from the bucket name - */ - protected DataLink findDataLink(String linkName) { - try { - String url = endpoint + "/data-links?status=AVAILABLE&search=" + linkName; - if (workspaceId != null) { - url += "&workspaceId=" + workspaceId; - } - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .GET() - .build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() < 200 && response.statusCode() >= 300) { - String message = response.body(); - if (message == null) message = ""; - message = message + " - HTTP " + response.statusCode(); - throw new RuntimeException("Failed to find data-link: " + linkName + message); - } - - JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); - JsonArray files = responseJson.getAsJsonArray("dataLinks"); - - List dataLinks = java.util.stream.StreamSupport.stream(files.spliterator(), false) - .map(JsonElement::getAsJsonObject) - .filter(obj -> obj.get("name").getAsString().equals(linkName)) - .map(obj -> getDataLinkFromJSONObject(obj)) - .toList(); - - if (dataLinks.isEmpty()) { - log.debug("No datalink response: " + linkName); - throw new RuntimeException("No Data-link found for " + linkName); - } - return dataLinks.getFirst(); - } catch (IOException e) { - throw new RuntimeException("Exception finding data-link", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Download interrupted", e); - } - } - - private DataLink getDataLinkFromJSONObject(JsonObject obj) { - String id = obj.get("id").getAsString(); - //Get first credential - String credentialsId = obj.get("credentials").getAsJsonArray().get(0).getAsJsonObject().get("id").getAsString(); - - return new DataLink(id, credentialsId); - } - - /** - * Download a file from the data-link - */ - protected byte[] downloadFile(String filePath) throws IOException { - try { - Map queryParams = new HashMap<>(); - if (workspaceId != null) { - queryParams.put("workspaceId", workspaceId); - } - if (dataLink.credentialsId != null) { - queryParams.put("credentialsId", dataLink.credentialsId); - } - - String url = buildDataLinkUrl("/download/"+ URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .GET() - .build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()); - - if (response.statusCode() == 404) { - log.debug("File {} in data-link {} not found", filePath, dataLink.id); - return null; - } - - if (response.statusCode() >= 400) { - throw new IOException("Failed to download file: " + filePath + " - status: " + response.statusCode()); - } - - return response.body(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Download interrupted", e); - } - } - - /** - * Download a file from the data-link directly to a local file - * This is more memory-efficient for large files like bundles - */ - protected void downloadFileToPath(String filePath, Path targetPath) throws IOException { - try { - Map queryParams = new HashMap<>(); - if (workspaceId != null) { - queryParams.put("workspaceId", workspaceId); - } - if (dataLink.credentialsId != null) { - queryParams.put("credentialsId", dataLink.credentialsId); - } - - String url = buildDataLinkUrl("/download/"+ URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .GET() - .build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofFile(targetPath)); - - if (response.statusCode() == 404) { - log.debug("File {} in data-link {} not found", filePath, dataLink.id); - throw new IOException("File not found: " + filePath); - } - - if (response.statusCode() >= 400) { - throw new IOException("Failed to download file: " + filePath + " - status: " + response.statusCode()); - } - - log.debug("Downloaded {} to {}", filePath, targetPath); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Download interrupted", e); - } - } - - /** - * Upload a file to the data-link - */ - protected void uploadFile(String filePath, Path localFile) throws IOException { - try { - // Step 1: Get upload URL - Map queryParams = new HashMap<>(); - if (workspaceId != null) { - queryParams.put("workspaceId", workspaceId); - } - if (dataLink.credentialsId != null) { - queryParams.put("credentialsId", dataLink.credentialsId); - } - String url = buildDataLinkUrl("/upload/" + URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); - - JsonObject uploadRequest = new JsonObject(); - uploadRequest.addProperty("fileName", localFile.getFileName().toString()); - uploadRequest.addProperty("contentLength", localFile.toFile().length()); - log.debug(" POST {}", url); - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(gson.toJson(uploadRequest))) - .build(); - - HttpResponse response = httpClient.sendAsString(request); - - if (response.statusCode() >= 400) { - String message = response.body(); - if (message == null) message = ""; - message = message + " - HTTP " + response.statusCode(); - throw new IOException("Failed to get upload URL for '" + filePath + "': " + message); - } - log.debug(response.body()); - JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); - String uploadId = responseJson.get("uploadId").getAsString(); - JsonArray jsonUploadUrls = responseJson.getAsJsonArray("uploadUrls").getAsJsonArray(); - List uploadUrls = StreamSupport.stream(jsonUploadUrls.spliterator(), false) - .map(JsonElement::getAsString) - .collect(java.util.stream.Collectors.toList()); - - for (String uploadUrl : uploadUrls) { - // Step 2: Upload file to the provided URL - HttpRequest uploadFileRequest = HttpRequest.newBuilder() - .uri(URI.create(uploadUrl)) - .header("Content-Type", "application/octet-stream") - .PUT(HttpRequest.BodyPublishers.ofFile(localFile)) - .build(); - - HttpResponse uploadResponse = httpClient.send(uploadFileRequest, HttpResponse.BodyHandlers.discarding()); - - if (uploadResponse.statusCode() >= 400) { - String message = response.body(); - if (message == null) message = ""; - message = message + " - HTTP " + response.statusCode(); - throw new IOException("Failed to upload file '" + filePath + "': " + message); - } - } - // Step 3: Finish upload (required for some providers like AWS) - queryParams = new HashMap<>(); - if (workspaceId != null) { - queryParams.put("workspaceId", workspaceId); - } - if (dataLink.credentialsId != null) { - queryParams.put("credentialsId", dataLink.credentialsId); - } - String finishUrl = buildDataLinkUrl("/upload/finish/"+ URLEncoder.encode(filePath, StandardCharsets.UTF_8), queryParams); - - JsonObject finishRequest = new JsonObject(); - finishRequest.addProperty("fileName", filePath); - finishRequest.addProperty("uploadId", filePath); - - HttpRequest finishReq = HttpRequest.newBuilder() - .uri(URI.create(finishUrl)) - .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(gson.toJson(finishRequest))) - .build(); - - httpClient.send(finishReq, HttpResponse.BodyHandlers.discarding()); - - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Upload interrupted", e); - } - } - - /** - * List files in a path - */ - protected List listFiles(String path) throws IOException { - Map queryParams = new HashMap<>(); - if (workspaceId != null) { - queryParams.put("workspaceId", workspaceId); - } - if (dataLink.credentialsId != null) { - queryParams.put("credentialsId", dataLink.credentialsId); - } - String url = buildDataLinkUrl("/browse/" + URLEncoder.encode(path, StandardCharsets.UTF_8), queryParams); - log.debug(" GET {}", url); - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .GET() - .build(); - - HttpResponse response = httpClient.sendAsString(request); - - if (response.statusCode() == 404) { - log.debug("No files found for {}", url); - return List.of(); - } - - if (response.statusCode() >= 400) { - log.debug("Error getting files {}", url); - String message = response.body(); - if (message == null) message = ""; - message = message + " - HTTP " + response.statusCode(); - - throw new IOException("Failed to list files in '" + path + "' : " + message); - } - log.debug(response.body()); - JsonObject responseJson = gson.fromJson(response.body(), JsonObject.class); - JsonArray files = responseJson.getAsJsonArray("objects"); - - return StreamSupport.stream(files.spliterator(), false) - .map(JsonElement::getAsJsonObject) - .map(obj -> obj.get("name").getAsString()) - .collect(java.util.stream.Collectors.toList()); - } - - /** - * Delete a file from the data-link - */ - protected void deleteFile(String filePath) throws IOException { - try { - Map queryParams = new HashMap<>(); - - if (workspaceId != null) { - queryParams.put("workspaceId", workspaceId); - } - String url = buildDataLinkUrl("/content", queryParams); - - JsonObject deleteRequest = new JsonObject(); - deleteRequest.addProperty("path", filePath); - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create(url)) - .header("Content-Type", "application/json") - .method("DELETE", HttpRequest.BodyPublishers.ofString(gson.toJson(deleteRequest))) - .build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.discarding()); - - if (response.statusCode() >= 400 && response.statusCode() != 404) { - throw new IOException("Failed to delete file: " + filePath + " - status: " + response.statusCode()); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Delete interrupted", e); - } - } - - private String buildDataLinkUrl(String path, Map queryParams) { - final StringBuilder url = new StringBuilder(endpoint + "/data-links/" + URLEncoder.encode(dataLink.id, StandardCharsets.UTF_8)); - if (path != null) { - if (!path.startsWith("/")) url.append("/"); - url.append(path); - } - - if (queryParams != null && !queryParams.isEmpty()) { - String queryString = queryParams.entrySet().stream() - .map(entry -> URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8) - + "=" - + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)) - .collect(Collectors.joining("&")); - url.append('?').append(queryString); - } - return url.toString(); - } - private void loadRefsMap() { log.debug("Loading refs Maps"); try { addRefs("heads"); } catch (Exception e) { - log.debug("No heads found for dataLink={} path={}: {}", dataLink.id, repoPath, e.getMessage()); + log.debug("No heads found for dataLink={} path={}: {}", dataLink.getId(), repoPath, e.getMessage()); } try { addRefs("tags"); } catch (Exception e) { - log.debug("No tags found for dataLink={} path={}: {}", dataLink.id, repoPath, e.getMessage()); + log.debug("No tags found for dataLink={} path={}: {}", dataLink.getId(), repoPath, e.getMessage()); } try { final String defaultBranch = getDefaultBranchRef(); @@ -449,22 +127,22 @@ private void loadRefsMap() { } } } catch (Exception e) { - log.debug("No default refs found for dataLink={} path={}: {}", dataLink.id, repoPath, e.getMessage()); + log.debug("No default refs found for dataLink={} path={}: {}", dataLink.getId(), repoPath, e.getMessage()); } } private void addRefs(String refType) throws IOException { String path = repoPath + "/refs/" + refType; - List branches = listFiles(path); + List branches = listFiles(httpClient, endpoint, dataLink, path, workspaceId); if (branches == null || branches.isEmpty()) { - log.debug("No {} refs found for dataLink={} path={}", refType, dataLink.id, repoPath); + log.debug("No {} refs found for dataLink={} path={}", refType, dataLink.getId(), repoPath); return; } for (String branch : branches) { String branchPath = path + "/" + branch; - List bundles = listFiles(branchPath); + List bundles = listFiles(httpClient, endpoint, dataLink, branchPath, workspaceId); if (bundles != null && !bundles.isEmpty()) { // Get the first bundle (there should only be one per branch) String bundleName = bundles.get(0); @@ -556,23 +234,4 @@ public ObjectId getObjectId() { return objectId; } } - - static class DataLink { - private String id; - private String credentialsId; - - private DataLink(String id, String credentialsId) { - this.id = id; - this.credentialsId = credentialsId; - } - - public String getId() { - return id; - } - - public String getCredentialsId() { - return credentialsId; - } - - } } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java index 104038596e..d6661c40c0 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraFetchConnection.java @@ -31,6 +31,9 @@ import java.nio.file.Path; import java.util.*; +import static io.seqera.tower.plugin.datalink.DataLinkUtils.downloadFile; +import static io.seqera.tower.plugin.datalink.DataLinkUtils.listFiles; + /** * Fetch Connection implementation for Seqera Platform data-links git-remote storage. * @@ -68,7 +71,7 @@ private void downloadBundle(Ref r, Set have, Path tmpdir, ProgressMoni // List bundles in the ref directory String refPath = repoPath + "/" + r.getName(); - List bundles = listFiles(refPath); + List bundles = listFiles(httpClient, endpoint, dataLink, refPath, workspaceId); if (bundles == null || bundles.isEmpty()) { throw new TransportException(transport.getURI(), "No bundle for " + r.getName()); @@ -86,7 +89,7 @@ private void downloadBundle(Ref r, Set have, Path tmpdir, ProgressMoni log.trace("Downloading bundle {} for branch {} to {}", bundlePath, r.getName(), localBundle); // Download the bundle - downloadFileToPath(bundlePath, localBundle); + downloadFile( httpClient, endpoint, dataLink, bundlePath, localBundle, workspaceId); parseBundle(r, have, localBundle, monitor); } diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java index 2693d64945..8d30e2668b 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraGitCredentialsProvider.java @@ -16,6 +16,7 @@ package io.seqera.tower.plugin.scm.jgit; +import io.seqera.tower.plugin.TowerRetryPolicy; import org.eclipse.jgit.errors.UnsupportedCredentialItem; import org.eclipse.jgit.transport.CredentialItem; import org.eclipse.jgit.transport.CredentialsProvider; @@ -31,6 +32,7 @@ public class SeqeraGitCredentialsProvider extends CredentialsProvider { private String endpoint; private String accessToken; private String workspaceId; + private TowerRetryPolicy retryPolicy; public void setEndpoint(String endpoint) { this.endpoint = endpoint; @@ -44,6 +46,8 @@ public void setWorkspaceId(String workspaceId) { this.workspaceId = workspaceId; } + public void setRetryPolicy(TowerRetryPolicy retryPolicy) {this.retryPolicy = retryPolicy; } + public String getEndpoint() { return endpoint != null ? endpoint : "https://api.cloud.seqera.io"; } @@ -59,6 +63,8 @@ public String getWorkspaceId() { return workspaceId; } + public TowerRetryPolicy getRetryPolicy() { return retryPolicy; } + @Override public boolean isInteractive() { return false; diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java index f53711f47a..a3c62685ed 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/scm/jgit/SeqeraPushConnection.java @@ -35,6 +35,8 @@ import java.util.List; import java.util.Map; +import static io.seqera.tower.plugin.datalink.DataLinkUtils.*; + /** * Push connection implementation for Seqera Platform data-links git-remote storage. * @@ -90,11 +92,11 @@ private void pushBranch(Map.Entry entry, Path tmpdir) t String bundlePath = repoPath + "/" + entry.getKey() + "/" + bundleFile.getFileName().toString(); log.trace("Uploading bundle {} to data-link {}", bundleFile, dataLink.getId()); - uploadFile(bundlePath, bundleFile); + uploadFile(httpClient, endpoint, dataLink, bundlePath, bundleFile, workspaceId ); if (oldBundlePath != null) { log.trace("Deleting old bundle {}", oldBundlePath); - deleteFile(oldBundlePath); + deleteFile(httpClient, endpoint, dataLink, oldBundlePath, workspaceId); } setUpdateStatus(entry.getValue(), RemoteRefUpdate.Status.OK); @@ -108,14 +110,16 @@ private void updateRemoteHead(String ref) { try { String headPath = repoPath + "/HEAD"; // Try to download HEAD to check if it exists - byte[] existingHead = downloadFile(headPath); + byte[] existingHead = getFileContent( httpClient, endpoint, dataLink, headPath, + workspaceId + ); if (existingHead == null) { log.debug("No remote default branch. Setting to {}.", ref); // Create a temporary file with the ref content Path tempFile = Files.createTempFile("head-", ".txt"); try { Files.write(tempFile, ref.getBytes(StandardCharsets.UTF_8)); - uploadFile(headPath, tempFile); + uploadFile(httpClient, endpoint, dataLink, headPath, tempFile, workspaceId); } finally { Files.deleteIfExists(tempFile); } @@ -166,7 +170,7 @@ public boolean isCommitInBranch(String bundlePath, Ref branchRef) throws IOExcep private String checkExistingBundle(String refName) throws IOException { String refPath = repoPath + "/" + refName; - List bundles = listFiles(refPath); + List bundles = listFiles(httpClient, endpoint, dataLink, refPath, workspaceId); if (bundles == null || bundles.isEmpty()) { return null; diff --git a/plugins/nf-tower/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/plugins/nf-tower/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider new file mode 100644 index 0000000000..d17b471f4b --- /dev/null +++ b/plugins/nf-tower/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -0,0 +1 @@ +io.seqera.tower.plugin.fs.SeqeraFileSystemProvider From 76f401a7e3dcc0091f908be5b0e5dbd31b2b7ba2 Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 6 Nov 2025 17:09:41 +0100 Subject: [PATCH 4/4] add push command Signed-off-by: jorgee --- .../main/groovy/nextflow/cli/CmdPush.groovy | 103 ++++++++++++++++++ .../main/groovy/nextflow/cli/Launcher.groovy | 1 + .../groovy/nextflow/scm}/PushManager.groovy | 3 +- .../plugin/launch/LaunchCommandImpl.groovy | 1 + 4 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 modules/nextflow/src/main/groovy/nextflow/cli/CmdPush.groovy rename {plugins/nf-tower/src/main/io/seqera/tower/plugin/launch => modules/nextflow/src/main/groovy/nextflow/scm}/PushManager.groovy (99%) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdPush.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdPush.groovy new file mode 100644 index 0000000000..0d27edfc67 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdPush.groovy @@ -0,0 +1,103 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cli +import com.beust.jcommander.Parameter +import com.beust.jcommander.Parameters +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.exception.AbortOperationException +import nextflow.plugin.Plugins +import nextflow.scm.AssetManager +import nextflow.scm.PushManager +import nextflow.util.TestOnly +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.transport.RemoteConfig +import org.eclipse.jgit.transport.URIish + + +/** + * CLI sub-command Push + * + * @author Jorge Ejarque + */ +@Slf4j +@CompileStatic +@Parameters(commandDescription = "Pushes a local implementation to a remote repository") +class CmdPush extends CmdBase implements HubOptions { + + static final public NAME = 'push' + + @Parameter(description = 'Repository URL to push to (optional if already configured as git remote)') + List args + + @Parameter(names=['-d', '-directory'], description = 'Local directory to push (default: current directory)') + String directory + + @Parameter(names=['-r','-revision'], description = 'Revision of the project to run (either a git branch, tag or commit SHA number)') + String revision = 'main' + + @Parameter(names=['-max-size'], description = 'Maximum file size in MB to push without confirmation (default: 10)') + int maxSizeMB = 10 + + @Parameter(names=['-message', '-m'], description = 'Commit message') + String message = 'Push from nextflow' + + @Override + final String getName() { NAME } + + @TestOnly + protected File root + + @Override + void run() { + if( args && args.size() > 1){ + throw new AbortOperationException('Incorrect number of arguments') + } + + // Get repository from args (optional) + def repository = args && args.size() == 1 ? args[0] : null + + // Folder defaults to current working directory if not specified + def folder = directory + ? new File(directory).getAbsoluteFile() + : new File(System.getProperty('user.dir')).getAbsoluteFile() + + if( !folder.exists() ) + throw new AbortOperationException("Folder does not exist: ${folder.absolutePath}") + + if( !folder.isDirectory() ) + throw new AbortOperationException("Path is not a directory: ${folder.absolutePath}") + + // init plugin system + Plugins.init() + + try { + final manager = new PushManager(folder) + def resolvedRepo = repository + if( !resolvedRepo ) { + resolvedRepo = manager.resolveRepository() + } + + log.info "Pushing folder ${folder.absolutePath} to repository ${resolvedRepo}" + manager.push(resolvedRepo, revision) + } + catch( Exception e ) { + throw new AbortOperationException("Failed to push folder: ${e.message}", e) + } + } + +} diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy index 95753b9757..b8296cbb9d 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy @@ -100,6 +100,7 @@ class Launcher { new CmdList(), new CmdLog(), new CmdPull(), + new CmdPush(), new CmdRun(), new CmdKubeRun(), new CmdDrop(), diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy b/modules/nextflow/src/main/groovy/nextflow/scm/PushManager.groovy similarity index 99% rename from plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy rename to modules/nextflow/src/main/groovy/nextflow/scm/PushManager.groovy index 6524cde8ca..8b3f1913bb 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/PushManager.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/scm/PushManager.groovy @@ -1,9 +1,8 @@ -package io.seqera.tower.plugin.launch +package nextflow.scm import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.exception.AbortOperationException -import nextflow.scm.AssetManager import org.eclipse.jgit.api.Git import org.eclipse.jgit.api.Status import org.eclipse.jgit.transport.RemoteConfig diff --git a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy b/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy index df18470cee..2929d91125 100644 --- a/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy +++ b/plugins/nf-tower/src/main/io/seqera/tower/plugin/launch/LaunchCommandImpl.groovy @@ -26,6 +26,7 @@ import io.seqera.tower.plugin.TowerClient import io.seqera.tower.plugin.datalink.DataLinkUtils import nextflow.BuildInfo import nextflow.cli.CmdLaunch +import nextflow.scm.PushManager import nextflow.util.ColorUtil import nextflow.exception.AbortOperationException import nextflow.file.FileHelper