diff --git a/wave-api/src/main/java/io/seqera/wave/api/PackagesSpec.java b/wave-api/src/main/java/io/seqera/wave/api/PackagesSpec.java index aa33e30..2b45b55 100644 --- a/wave-api/src/main/java/io/seqera/wave/api/PackagesSpec.java +++ b/wave-api/src/main/java/io/seqera/wave/api/PackagesSpec.java @@ -21,6 +21,7 @@ import java.util.Objects; import io.seqera.wave.config.CondaOpts; +import io.seqera.wave.config.PixiOpts; /** * Model a Package environment requirements @@ -48,6 +49,11 @@ public enum Type { CONDA } */ public CondaOpts condaOpts; + /** + * Pixi build options + */ + public PixiOpts pixiOpts; + /** * channels used for downloading packages */ @@ -62,6 +68,7 @@ public boolean equals(Object object) { && Objects.equals(environment, that.environment) && Objects.equals(entries, that.entries) && Objects.equals(condaOpts, that.condaOpts) + && Objects.equals(pixiOpts, that.pixiOpts) && Objects.equals(channels, that.channels); } @@ -77,6 +84,7 @@ public String toString() { ", envFile='" + environment + '\'' + ", packages=" + entries + ", condaOpts=" + condaOpts + + ", pixiOpts=" + pixiOpts + ", channels=" + ObjectUtils.toString(channels) + '}'; } @@ -106,4 +114,8 @@ public PackagesSpec withCondaOpts(CondaOpts opts) { return this; } + public PackagesSpec withPixiOpts(PixiOpts opts) { + this.pixiOpts = opts; + return this; + } } diff --git a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java index 79f3c94..1679fe0 100644 --- a/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java +++ b/wave-api/src/main/java/io/seqera/wave/api/SubmitContainerTokenRequest.java @@ -124,7 +124,7 @@ public class SubmitContainerTokenRequest implements Cloneable { public String workflowId; /** - * One or more container should be included in upstream container request + * One or more container should be included in the upstream container request */ public List containerIncludes; @@ -150,7 +150,7 @@ public class SubmitContainerTokenRequest implements Cloneable { public ScanMode scanMode; /** - * Define the allows security vulnerabilities in the container request. + * Define the allowed security vulnerabilities in the container request. * Empty or null means no vulnerabilities are allowed. */ public List scanLevels; @@ -160,6 +160,11 @@ public class SubmitContainerTokenRequest implements Cloneable { */ public BuildCompression buildCompression; + /** + * The build template that should be used to build the container image. + */ + public String buildTemplate; + public SubmitContainerTokenRequest copyWith(Map opts) { try { final SubmitContainerTokenRequest copy = (SubmitContainerTokenRequest) this.clone(); @@ -213,6 +218,8 @@ public SubmitContainerTokenRequest copyWith(Map opts) { copy.scanLevels = (List) opts.get("scanLevels"); if( opts.containsKey("buildCompression")) copy.buildCompression = (BuildCompression) opts.get("buildCompression"); + if( opts.containsKey("buildTemplate")) + copy.buildTemplate = (String) opts.get("buildTemplate"); // done return copy; } @@ -353,6 +360,11 @@ public SubmitContainerTokenRequest withBuildCompression(BuildCompression mode) { return this; } + public SubmitContainerTokenRequest withBuildTemplate(String template) { + this.buildTemplate = template; + return this; + } + public boolean formatSingularity() { return "sif".equals(format); } @@ -385,6 +397,7 @@ public String toString() { ", scanMode=" + scanMode + ", scanLevels=" + scanLevels + ", buildCompression=" + buildCompression + + ", buildTemplate=" + buildTemplate + '}'; } } diff --git a/wave-api/src/main/java/io/seqera/wave/config/CondaOpts.java b/wave-api/src/main/java/io/seqera/wave/config/CondaOpts.java index a4203b5..66cc503 100644 --- a/wave-api/src/main/java/io/seqera/wave/config/CondaOpts.java +++ b/wave-api/src/main/java/io/seqera/wave/config/CondaOpts.java @@ -28,11 +28,18 @@ */ public class CondaOpts { final public static String DEFAULT_MAMBA_IMAGE = "mambaorg/micromamba:1.5.10-noble"; + final public static String DEFAULT_MAMBA_IMAGE_V2 = "mambaorg/micromamba:2.1.1"; final public static String DEFAULT_PACKAGES = "conda-forge::procps-ng"; + final public static String DEFAULT_BASE_IMAGE = "ubuntu:24.04"; public String mambaImage; public List commands; public String basePackages; + public String baseImage; + + static public CondaOpts v2() { + return new CondaOpts(Map.of("mambaImage", DEFAULT_MAMBA_IMAGE_V2)); + } public CondaOpts() { this(Map.of()); @@ -41,6 +48,7 @@ public CondaOpts(Map opts) { this.mambaImage = opts.containsKey("mambaImage") ? opts.get("mambaImage").toString(): DEFAULT_MAMBA_IMAGE; this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; this.basePackages = opts.containsKey("basePackages") ? (String)opts.get("basePackages") : DEFAULT_PACKAGES; + this.baseImage = opts.containsKey("baseImage") ? opts.get("baseImage").toString(): DEFAULT_BASE_IMAGE; } public CondaOpts withMambaImage(String value) { @@ -60,10 +68,11 @@ public CondaOpts withBasePackages(String value) { @Override public String toString() { - return String.format("CondaOpts(mambaImage=%s; basePackages=%s, commands=%s)", + return String.format("CondaOpts(mambaImage=%s; basePackages=%s, commands=%s, baseImage=%s)", mambaImage, basePackages, - commands != null ? String.join(",", commands) : "null" + commands != null ? String.join(",", commands) : "null", + baseImage ); } @@ -72,11 +81,15 @@ public boolean equals(Object object) { if (this == object) return true; if (object == null || getClass() != object.getClass()) return false; CondaOpts condaOpts = (CondaOpts) object; - return Objects.equals(mambaImage, condaOpts.mambaImage) && Objects.equals(commands, condaOpts.commands) && Objects.equals(basePackages, condaOpts.basePackages); + return Objects.equals(mambaImage, condaOpts.mambaImage) + && Objects.equals(commands, condaOpts.commands) + && Objects.equals(basePackages, condaOpts.basePackages) + && Objects.equals(baseImage, condaOpts.baseImage) + ; } @Override public int hashCode() { - return Objects.hash(mambaImage, commands, basePackages); + return Objects.hash(mambaImage, commands, basePackages, baseImage); } } diff --git a/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java b/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java new file mode 100644 index 0000000..4717fb2 --- /dev/null +++ b/wave-api/src/main/java/io/seqera/wave/config/PixiOpts.java @@ -0,0 +1,91 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.config; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * @author Paolo Di Tommaso + */ +public class PixiOpts { + + final public static String DEFAULT_PIXI_IMAGE = "ghcr.io/prefix-dev/pixi:latest"; + final public static String DEFAULT_BASE_IMAGE = "ubuntu:24.04"; + final public static String DEFAULT_PACKAGES = "conda-forge::procps-ng"; + + public String pixiImage; + public List commands; + public String basePackages; + public String baseImage; + + public PixiOpts() { + this(Map.of()); + } + public PixiOpts(Map opts) { + this.pixiImage = opts.containsKey("pixiImage") ? opts.get("pixiImage").toString(): DEFAULT_PIXI_IMAGE; + this.baseImage = opts.containsKey("baseImage") ? opts.get("baseImage").toString(): DEFAULT_BASE_IMAGE; + this.commands = opts.containsKey("commands") ? (List)opts.get("commands") : null; + this.basePackages = opts.containsKey("basePackages") ? (String)opts.get("basePackages") : DEFAULT_PACKAGES; + } + + public PixiOpts withMambaImage(String value) { + this.pixiImage = value; + return this; + } + + public PixiOpts withCommands(List value) { + this.commands = value; + return this; + } + + public PixiOpts withBasePackages(String value) { + this.basePackages = value; + return this; + } + + @Override + public String toString() { + return String.format("PixiOpts(pixiImage=%s; basePackages=%s, commands=%s, baseImage=%s)", + pixiImage, + basePackages, + commands != null ? String.join(",", commands) : "null", + baseImage + ); + } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null || getClass() != object.getClass()) return false; + CondaOpts condaOpts = (CondaOpts) object; + return Objects.equals(pixiImage, condaOpts.mambaImage) + && Objects.equals(commands, condaOpts.commands) + && Objects.equals(basePackages, condaOpts.basePackages) + && Objects.equals(baseImage, condaOpts.baseImage) + ; + } + + @Override + public int hashCode() { + return Objects.hash(pixiImage, commands, basePackages, baseImage); + } + +} diff --git a/wave-api/src/test/groovy/io/seqera/wave/api/SubmitContainerTokenRequestTest.groovy b/wave-api/src/test/groovy/io/seqera/wave/api/SubmitContainerTokenRequestTest.groovy index 0e03283..792cbea 100644 --- a/wave-api/src/test/groovy/io/seqera/wave/api/SubmitContainerTokenRequestTest.groovy +++ b/wave-api/src/test/groovy/io/seqera/wave/api/SubmitContainerTokenRequestTest.groovy @@ -52,6 +52,7 @@ class SubmitContainerTokenRequestTest extends Specification { scanMode: ScanMode.async, scanLevels: List.of(ScanLevel.LOW, ScanLevel.MEDIUM), buildCompression: BuildCompression.gzip, + buildTemplate: 'micromamba', ) when: @@ -81,6 +82,7 @@ class SubmitContainerTokenRequestTest extends Specification { copy.scanMode == req.scanMode copy.scanLevels == req.scanLevels copy.buildCompression == req.buildCompression + copy.buildTemplate == req.buildTemplate and: copy.formatSingularity() @@ -110,6 +112,7 @@ class SubmitContainerTokenRequestTest extends Specification { scanMode: ScanMode.required, scanLevels: List.of(ScanLevel.HIGH), buildCompression: BuildCompression.estargz, + buildTemplate: 'pixi', ) then: other.towerAccessToken == 'b1' @@ -135,6 +138,7 @@ class SubmitContainerTokenRequestTest extends Specification { other.scanMode == ScanMode.required other.scanLevels == List.of(ScanLevel.HIGH) other.buildCompression == BuildCompression.estargz + other.buildTemplate == 'pixi' and: !other.formatSingularity() } diff --git a/wave-utils/src/main/java/io/seqera/wave/util/DockerHelper.java b/wave-utils/src/main/java/io/seqera/wave/util/DockerHelper.java index 7c14cca..20a3524 100644 --- a/wave-utils/src/main/java/io/seqera/wave/util/DockerHelper.java +++ b/wave-utils/src/main/java/io/seqera/wave/util/DockerHelper.java @@ -33,6 +33,7 @@ import java.util.stream.Collectors; import io.seqera.wave.config.CondaOpts; +import io.seqera.wave.config.PixiOpts; import org.apache.commons.lang3.StringUtils; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.Yaml; @@ -224,6 +225,23 @@ static public String condaPackagesToSingularityFile(String packages, List condaChannels, CondaOpts opts) { + return condaPackagesTemplate0( + "/templates/micromamba/dockerfile-conda-packages.txt", + packages, + condaChannels, + opts); + } + + static public String condaPackagesToSingularityFileUsingMicromamba(String packages, List condaChannels, CondaOpts opts) { + return condaPackagesTemplate0( + "/templates/micromamba/singularityfile-conda-packages.txt", + packages, + condaChannels, + opts); + } + + @Deprecated static protected String condaPackagesTemplate0(String template, String packages, List condaChannels, CondaOpts opts) { final List channels0 = condaChannels!=null ? condaChannels : List.of(); final String channelsOpts = channels0.stream().map(it -> "-c "+it).collect(Collectors.joining(" ")); @@ -242,6 +260,23 @@ static protected String condaPackagesTemplate0(String template, String packages, return addCommands(result, opts.commands, singularity); } + static protected String condaPackagesTemplate1(String template, String packages, List condaChannels, CondaOpts opts) { + final List channels0 = condaChannels!=null ? condaChannels : List.of(); + final String channelsOpts = channels0.stream().map(it -> "-c "+it).collect(Collectors.joining(" ")); + final boolean singularity = template.contains("/singularityfile"); + final String target = packages.startsWith("http://") || packages.startsWith("https://") + ? "-f " + packages + : packages; + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("mamba_image", opts.mambaImage); + binding.put("channel_opts", channelsOpts); + binding.put("target", target); + binding.put("base_packages", mambaInstallBasePackage0(opts.basePackages,singularity)); + + final String result = renderTemplate0(template, binding) ; + return addCommands(result, opts.commands, singularity); + } static public String condaFileToDockerFile(CondaOpts opts) { return condaFileTemplate0("/templates/conda/dockerfile-conda-file.txt", opts); @@ -251,6 +286,22 @@ static public String condaFileToSingularityFile(CondaOpts opts) { return condaFileTemplate0("/templates/conda/singularityfile-conda-file.txt", opts); } + static public String condaFileToDockerFileUsingMicromamba(CondaOpts opts) { + return condaFileTemplate0("/templates/micromamba/dockerfile-conda-file.txt", opts); + } + + static public String condaFileToSingularityFileUsingMicromamba(CondaOpts opts) { + return condaFileTemplate0("/templates/micromamba/singularityfile-conda-file.txt", opts); + } + + static public String condaFileToDockerFileUsingPixi(PixiOpts opts) { + return condaFileTemplate1("/templates/pixi-v1/dockerfile-conda-file.txt", opts); + } + + static public String condaFileToSingularityFileUsingPixi(PixiOpts opts) { + return condaFileTemplate1("/templates/pixi-v1/singularityfile-conda-file.txt", opts); + } + static protected String condaFileTemplate0(String template, CondaOpts opts) { final boolean singularity = template.contains("/singularityfile"); // create the binding map @@ -262,6 +313,30 @@ static protected String condaFileTemplate0(String template, CondaOpts opts) { return addCommands(result, opts.commands, singularity); } + static protected String condaFileTemplate2(String template, CondaOpts opts) { + final boolean singularity = template.contains("/singularityfile"); + // create the binding map + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("mamba_image", opts.mambaImage); + binding.put("base_packages", mambaInstallBasePackage0(opts.basePackages,singularity)); + + final String result = renderTemplate0(template, binding, List.of("wave_context_dir")); + return addCommands(result, opts.commands, singularity); + } + + static protected String condaFileTemplate1(String template, PixiOpts opts) { + final boolean singularity = template.contains("/singularityfile"); + // create the binding map + final Map binding = new HashMap<>(); + binding.put("base_image", opts.baseImage); + binding.put("pixi_image", opts.pixiImage); + binding.put("base_packages", pixiAddBasePackage0(opts.basePackages,singularity)); + + final String result = renderTemplate0(template, binding, List.of("wave_context_dir")); + return addCommands(result, opts.commands, singularity); + } + static private String renderTemplate0(String templatePath, Map binding) { return renderTemplate0(templatePath, binding, List.of()); } @@ -290,6 +365,15 @@ private static String mambaInstallBasePackage0(String basePackages, boolean sing : "&& " + result + " \\"; } + private static String pixiAddBasePackage0(String basePackages, boolean singularity) { + String result = !StringUtils.isEmpty(basePackages) + ? String.format("pixi add %s", basePackages) + : null; + return result==null || singularity + ? result + : "&& " + result + " \\"; + } + static private String addCommands(String result, List commands, boolean singularity) { if( commands==null || commands.isEmpty() ) return result; diff --git a/wave-utils/src/main/resources/templates/micromamba-v2/dockerfile-conda-file.txt b/wave-utils/src/main/resources/templates/micromamba-v2/dockerfile-conda-file.txt new file mode 100644 index 0000000..12a0ef6 --- /dev/null +++ b/wave-utils/src/main/resources/templates/micromamba-v2/dockerfile-conda-file.txt @@ -0,0 +1,15 @@ +FROM {{mamba_image}}:2.1.1 AS build +COPY --chown=$MAMBA_USER:$MAMBA_USER conda.yml /tmp/conda.yml +RUN micromamba install -y -n base -f /tmp/conda.yml \ + {{base_packages}} + && micromamba env export --name base --explicit > environment.lock \ + && echo ">> CONDA_LOCK_START" \ + && cat environment.lock \ + && echo "<< CONDA_LOCK_END" + +FROM {{base_image}} AS prod +ARG MAMBA_ROOT_PREFIX="/opt/conda" +ENV MAMBA_ROOT_PREFIX=$MAMBA_ROOT_PREFIX +COPY --from=build "$MAMBA_ROOT_PREFIX" "$MAMBA_ROOT_PREFIX" +USER root +ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" diff --git a/wave-utils/src/main/resources/templates/micromamba-v2/dockerfile-conda-packages.txt b/wave-utils/src/main/resources/templates/micromamba-v2/dockerfile-conda-packages.txt new file mode 100644 index 0000000..51d48f0 --- /dev/null +++ b/wave-utils/src/main/resources/templates/micromamba-v2/dockerfile-conda-packages.txt @@ -0,0 +1,15 @@ +FROM {{mamba_image}}:2.1.1 AS build +RUN \ + micromamba install -y -n base {{channel_opts}} {{target}} \ + {{base_packages}} + && micromamba env export --name base --explicit > environment.lock \ + && echo ">> CONDA_LOCK_START" \ + && cat environment.lock \ + && echo "<< CONDA_LOCK_END" + +FROM {{base_image}} AS prod +ARG MAMBA_ROOT_PREFIX="/opt/conda" +ENV MAMBA_ROOT_PREFIX=$MAMBA_ROOT_PREFIX +COPY --from=build "$MAMBA_ROOT_PREFIX" "$MAMBA_ROOT_PREFIX" +USER root +ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" diff --git a/wave-utils/src/main/resources/templates/pixi-v1/dockerfile-conda-file.txt b/wave-utils/src/main/resources/templates/pixi-v1/dockerfile-conda-file.txt new file mode 100644 index 0000000..d98f2be --- /dev/null +++ b/wave-utils/src/main/resources/templates/pixi-v1/dockerfile-conda-file.txt @@ -0,0 +1,25 @@ +FROM {{pixi_image}} AS build + +COPY conda.yml /opt/wave/conda.yml +WORKDIR /opt/wave + +RUN pixi init --import /opt/wave/conda.yml \ + {{base_packages}} + && pixi shell-hook > /shell-hook.sh \ + && echo 'exec "$@"' >> /shell-hook.sh \ + && echo ">> CONDA_LOCK_START" \ + && cat /opt/wave/pixi.lock \ + && echo "<< CONDA_LOCK_END" + +FROM {{base_image}} AS final + +# copy the pixi environment in the final container +COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default +COPY --from=build /shell-hook.sh /shell-hook.sh + +# set the entrypoint to the shell-hook script (activate the environment and run the command) +# no more pixi needed in the final container +ENTRYPOINT ["/bin/bash", "/shell-hook.sh"] + +# Default command for "docker run" +CMD ["/bin/bash"] diff --git a/wave-utils/src/main/resources/templates/pixi-v1/singularityfile-conda-file.txt b/wave-utils/src/main/resources/templates/pixi-v1/singularityfile-conda-file.txt new file mode 100644 index 0000000..a735a78 --- /dev/null +++ b/wave-utils/src/main/resources/templates/pixi-v1/singularityfile-conda-file.txt @@ -0,0 +1,24 @@ +BootStrap: docker +From: {{pixi_image}} +Stage: build +%files + {{wave_context_dir}}/conda.yml /scratch/conda.yml +%post + mkdir /opt/wave && cd /opt/wave + pixi init --import /scratch/conda.yml + {{base_packages}} + pixi shell-hook > /shell-hook.sh + echo 'exec "$@"' >> /shell-hook.sh + echo ">> CONDA_LOCK_START" + cat /opt/wave/pixi.lock + echo "<< CONDA_LOCK_END" + +Bootstrap: docker +From: {{base_image}} +Stage: final +# install binary from stage one +%files from build + /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default + /shell-hook.sh /shell-hook.sh +%environment + export PATH="$MAMBA_ROOT_PREFIX/bin:$PATH" diff --git a/wave-utils/src/test/groovy/io/seqera/wave/util/DockerHelperTest.groovy b/wave-utils/src/test/groovy/io/seqera/wave/util/DockerHelperTest.groovy index 6096ddb..eab5389 100644 --- a/wave-utils/src/test/groovy/io/seqera/wave/util/DockerHelperTest.groovy +++ b/wave-utils/src/test/groovy/io/seqera/wave/util/DockerHelperTest.groovy @@ -21,7 +21,7 @@ package io.seqera.wave.util import java.nio.file.Files import io.seqera.wave.config.CondaOpts - +import io.seqera.wave.config.PixiOpts import spock.lang.Specification /** * @@ -683,4 +683,37 @@ class DockerHelperTest extends Specification { '''.stripIndent() } + def 'should create dockerfile content from conda file using pixi' () { + given: + def PIXI_OPTS = new PixiOpts([basePackages: 'foo::bar']) + + expect: + DockerHelper.condaFileToDockerFileUsingPixi(PIXI_OPTS)== '''\ + FROM ghcr.io/prefix-dev/pixi:latest AS build + + COPY conda.yml /opt/wave/conda.yml + WORKDIR /opt/wave + + RUN pixi init --import /opt/wave/conda.yml \\ + && pixi add foo::bar \\ + && pixi shell-hook > /shell-hook.sh \\ + && echo 'exec "$@"' >> /shell-hook.sh \\ + && echo ">> CONDA_LOCK_START" \\ + && cat /opt/wave/pixi.lock \\ + && echo "<< CONDA_LOCK_END" + + FROM ubuntu:24.04 AS prod + + # copy the pixi environment in the final container + COPY --from=build /opt/wave/.pixi/envs/default /opt/wave/.pixi/envs/default + COPY --from=build /shell-hook.sh /shell-hook.sh + + # set the entrypoint to the shell-hook script (activate the environment and run the command) + # no more pixi needed in the prod container + ENTRYPOINT ["/bin/bash", "/shell-hook.sh"] + + # Default command for "docker run" + CMD ["/bin/bash"] + '''.stripIndent() + } }