diff --git a/.github/workflows/create_workflows.py b/.github/workflows/create_workflows.py index 646a4d5e..b2f85b74 100644 --- a/.github/workflows/create_workflows.py +++ b/.github/workflows/create_workflows.py @@ -31,12 +31,12 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape apt_based = [ + "ubuntu:24.04", "ubuntu:22.04", - "ubuntu:18.04", + "debian:bookworm-slim", "debian:bullseye-slim", - "debian:buster-slim", ] -yum_based = ["fedora:36", "centos:7"] +yum_based = ["fedora:40", "centos:8"] """ Add a "skip_versions" key to the software dictionary if you want to skip diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 2bc63178..a764f1b7 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install Apptainer env: - VERSION: 1.1.5 + VERSION: 1.3.6 run: | sudo apt-get update sudo apt-get install -y wget diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 974c9154..c4599464 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,4 +41,4 @@ jobs: run: | ~/auto shipit -vv env: - GH_TOKEN: ${{ secrets.AUTO_USER_TOKEN }} + GH_TOKEN: ${{ secrets.AUTO_TOKEN }} diff --git a/docs/user_guide/examples.rst b/docs/user_guide/examples.rst index 62f8ea3a..5685384e 100644 --- a/docs/user_guide/examples.rst +++ b/docs/user_guide/examples.rst @@ -117,7 +117,7 @@ Docker neurodocker generate docker \ --pkg-manager yum \ - --base-image fedora:36 \ + --base-image fedora:40 \ --afni method=binaries version=latest \ > afni-binaries.Dockerfile @@ -130,7 +130,7 @@ This does not install AFNI's R packages. To install relevant R things, use the f neurodocker generate docker \ --pkg-manager yum \ - --base-image fedora:36 \ + --base-image fedora:40 \ --afni method=binaries version=latest install_r_pkgs=true \ > afni-binaries-r.Dockerfile @@ -149,7 +149,7 @@ This does not install AFNI's R packages. To install relevant R things, use the f .. neurodocker generate docker \ .. --pkg-manager yum \ -.. --base-image fedora:36 \ +.. --base-image fedora:40 \ .. --afni method=source version=master \ .. > afni-source.Dockerfile diff --git a/neurodocker/cli/minify/_trace.sh b/neurodocker/cli/minify/_trace.sh index f9f04861..7dd0c9d2 100644 --- a/neurodocker/cli/minify/_trace.sh +++ b/neurodocker/cli/minify/_trace.sh @@ -38,9 +38,9 @@ function install_missing_dependencies() { function install_conda_reprozip() { - TMP_CONDA_INSTALLER=/tmp/miniconda.sh + TMP_CONDA_INSTALLER=/tmp/miniforge.sh ls /tmp - curl -sSL -o "$TMP_CONDA_INSTALLER" "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" + curl -sSL -o "$TMP_CONDA_INSTALLER" "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" bash $TMP_CONDA_INSTALLER -b -f -p $REPROZIP_CONDA rm -f $TMP_CONDA_INSTALLER ${REPROZIP_CONDA}/bin/mamba install -c conda-forge -y reprozip diff --git a/neurodocker/cli/minify/tests/test_minify.py b/neurodocker/cli/minify/tests/test_minify.py index ee8859c8..a9298ab2 100644 --- a/neurodocker/cli/minify/tests/test_minify.py +++ b/neurodocker/cli/minify/tests/test_minify.py @@ -16,7 +16,7 @@ @skip_arm_on_mac def test_minify(): client = docker.from_env() - container = client.containers.run("python:3.9-slim", detach=True, tty=True) + container = client.containers.run("python:3.10-slim", detach=True, tty=True) commands = ["python --version", """python -c 'print(123)'"""] try: runner = CliRunner() @@ -35,7 +35,8 @@ def test_minify(): assert ret == 0, f"unexpected non-zero return code when running '{cmd}'" # This should fail. - ret, result = container.exec_run("pip --help") + cmd = "pip --help" + ret, result = container.exec_run(cmd) assert ret != 0, f"unexpected zero return code when running '{cmd}'" finally: @@ -46,7 +47,7 @@ def test_minify(): @skip_arm_on_mac def test_minify_abort(): client = docker.from_env() - container = client.containers.run("python:3.9-slim", detach=True, tty=True) + container = client.containers.run("python:3.10-slim", detach=True, tty=True) commands = ["python --version", """python -c 'print(123)'"""] try: runner = CliRunner() @@ -64,7 +65,8 @@ def test_minify_abort(): assert ret == 0, f"unexpected non-zero return code when running '{cmd}'" # This should still succeed. - ret, result = container.exec_run("pip --help") + cmd = "pip --help" + ret, result = container.exec_run(cmd) assert ret == 0, f"unexpected non-zero return code when running '{cmd}'" finally: @@ -80,7 +82,7 @@ def test_minify_with_mounted_volume(tmp_path: Path): (tmp_path / "foobar.txt").write_text("Foobar") container = client.containers.run( - "python:3.8-slim", + "python:3.10-slim", detach=True, tty=True, volumes={str(tmp_path): {"bind": "/work", "mode": "rw"}}, diff --git a/neurodocker/cli/tests/sample-template-jq.yaml b/neurodocker/cli/tests/sample-template-jq.yaml index 25a89ccc..86117d53 100644 --- a/neurodocker/cli/tests/sample-template-jq.yaml +++ b/neurodocker/cli/tests/sample-template-jq.yaml @@ -17,12 +17,17 @@ binaries: ND_BAZ: '1234' ND_FOO: BAR instructions: | + export DUMMY="dummy" + {%- if self.pkg_manager == "yum" %} + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + {%- endif %} {{ self.install_dependencies() }} curl -fsSL --output /usr/local/bin/jq {{ self.urls[self.version]}} chmod +x /usr/local/bin/jq urls: - '1.5': https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 - '1.6': https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 + '1.6': https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 + '1.7': https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64 source: arguments: required: @@ -48,13 +53,21 @@ source: ND_BAZ: '1234' ND_FOO: BAR instructions: | + export DUMMY="dummy" + {%- if self.pkg_manager == "yum" %} + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + {%- endif %} {{ self.install_dependencies() }} mkdir jq cd jq - curl -fsSL https://github.com/stedolan/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \ + curl -fsSL https://github.com/jqlang/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \ | tar xz --strip-components 1 - # Only required in version 1.6 - autoreconf -fi - ./configure --disable-maintainer-mode + {%- if self.version == '1.6' %} + autoreconf -i + ./configure --with-oniguruma=builtin --disable-maintainer-mode + {%- else %} + ./configure --with-oniguruma=builtin + {%- endif %} make make install diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index f5444535..b277eb59 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -26,7 +26,8 @@ ], ) @pytest.mark.parametrize( - ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "centos:7")] + ["pkg_manager", "base_image"], + [("apt", "debian:bullseye-slim"), ("yum", "centos:8")], ) def test_build_image_from_registered( tmp_path: Path, cmd: str, pkg_manager: str, base_image: str @@ -46,11 +47,11 @@ def test_build_image_from_registered( "--pkg-manager", pkg_manager, "--jq", - "version=1.5", + "version=1.7", ], ) assert result.exit_code == 0, result.output - assert "jq-1.5/jq-linux64" in result.output + assert "jq-1.7/jq-linux64" in result.output spec = "Dockerfile" if cmd == "docker" else "Singularity" (tmp_path / spec).write_text(result.output) @@ -59,6 +60,8 @@ def test_build_image_from_registered( with build_fn(tmp_path) as img: stdout, _ = run_fn(img, args=["jq", "--help"]) assert "jq is a tool for processing JSON" in stdout + stdout, _ = run_fn(img, args=["jq", "--version"]) + assert "jq-1.7" in stdout @pytest.mark.long @@ -82,7 +85,7 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path): cmd, "--json", "--base-image", - "debian:buster-slim", + "debian:bullseye-slim", "--pkg-manager", "apt", "--install", diff --git a/neurodocker/cli/tests/test_cli.py b/neurodocker/cli/tests/test_cli.py index 88dcf475..05df6e6c 100644 --- a/neurodocker/cli/tests/test_cli.py +++ b/neurodocker/cli/tests/test_cli.py @@ -165,18 +165,18 @@ def test_render_registered(cmd: str, pkg_manager: str): [ cmd, "--base-image", - "debian:buster", + "debian:bullseye", "--pkg-manager", pkg_manager, "--jq", - "version=1.5", - "--jq", "version=1.6", + "--jq", + "version=1.7", ], ) assert result.exit_code == 0, result.output - assert "jq-1.5/jq-linux64" in result.output assert "jq-1.6/jq-linux64" in result.output + assert "jq-1.7/jq-linux64" in result.output # Test that we add the default header and default/custom entrypoints @@ -188,7 +188,7 @@ def test_default_header_and_entrypoint(cmd: str, pkg_manager: str, entrypoint: s cmd_ = [ cmd, "--base-image", - "debian:buster", + "debian:bullseye", "--pkg-manager", pkg_manager, ] diff --git a/neurodocker/reproenv/tests/sample-template-jq.yaml b/neurodocker/reproenv/tests/sample-template-jq.yaml index 629ab469..f6fdfc70 100644 --- a/neurodocker/reproenv/tests/sample-template-jq.yaml +++ b/neurodocker/reproenv/tests/sample-template-jq.yaml @@ -12,7 +12,7 @@ binaries: - ca-certificates - curl debs: - # Not required for jq -- used to test if packackages are installed properly with dpkg. + # Not required for jq -- used to test if packages are installed properly with dpkg. - http://ftp.us.debian.org/debian/pool/main/r/rust-fd-find/fd-find_7.2.0-2_amd64.deb yum: - curl @@ -25,8 +25,8 @@ binaries: curl -fsSL --output /usr/local/bin/jq {{ self.urls[self.version]}} chmod +x /usr/local/bin/jq urls: - '1.5': https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 - '1.6': https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 + '1.6': https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 + '1.7': https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64 source: arguments: required: @@ -55,10 +55,13 @@ source: {{ self.install_dependencies() }} mkdir jq cd jq - curl -fsSL https://github.com/stedolan/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \ + curl -fsSL https://github.com/jqlang/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \ | tar xz --strip-components 1 - # Only required in version 1.6 - autoreconf -fi - ./configure --disable-maintainer-mode + {%- if self.version == '1.6' %} + autoreconf -i + ./configure --with-oniguruma=builtin --disable-maintainer-mode + {%- else %} + ./configure --with-oniguruma=builtin + {%- endif %} make make install diff --git a/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py b/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py index b616af46..faae9f94 100644 --- a/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py +++ b/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py @@ -26,7 +26,8 @@ ], ) @pytest.mark.parametrize( - ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "fedora:33")] + ["pkg_manager", "base_image"], + [("apt", "debian:bullseye-slim"), ("yum", "fedora:40")], ) @pytest.mark.parametrize( ["jq_version", "jq_version_output", "fd_version_startswith"], @@ -81,12 +82,13 @@ def test_build_using_renderer_from_dict( ], ) @pytest.mark.parametrize( - ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "fedora:33")] + ["pkg_manager", "base_image"], + [("apt", "debian:bullseye-slim"), ("yum", "fedora:40")], ) @pytest.mark.parametrize(["method"], [("binaries",), ("source",)]) @pytest.mark.parametrize( ["jq_version", "jq_version_output", "fd_version_startswith"], - [("1.6", "jq-1.6", "fd"), ("1.5", "jq-1.5", "fd")], + [("1.6", "jq-1.6", "fd"), ("1.7", "jq-1.7", "fd")], ) def test_build_using_renderer_instance_methods( cmd: str, @@ -118,10 +120,7 @@ def test_build_using_renderer_instance_methods( stdout, _ = run_fn(img, args=["jq", "--help"]) assert stdout.startswith("jq - commandline JSON processor") stdout, _ = run_fn(img, args=["jq", "--version"]) - if method == "source" and jq_version == "1.5": - assert stdout == "jq-" - else: - assert stdout == jq_version_output + assert stdout == jq_version_output # Test that deb was installed if method == "binaries": stdout, _ = run_fn(img, args=[fd_exe, "--version"]) diff --git a/neurodocker/reproenv/tests/test_build_images_simple.py b/neurodocker/reproenv/tests/test_build_images_simple.py index 535e8a82..c9039f52 100644 --- a/neurodocker/reproenv/tests/test_build_images_simple.py +++ b/neurodocker/reproenv/tests/test_build_images_simple.py @@ -21,9 +21,9 @@ def test_build_simple(cmd: str, tmp_path): # Create a Dockerfile. r = rcls("apt") if isinstance(r, DockerRenderer): - r.from_("debian:buster-slim", as_="builder") + r.from_("debian:bullseye-slim", as_="builder") else: - r.from_("debian:buster-slim") + r.from_("debian:bullseye-slim") r.arg("FOO") r.copy(["foo.txt", "tst/baz.txt"], "/opt/") r.env(PATH="$PATH:/opt/foo/bin") diff --git a/neurodocker/reproenv/tests/test_renderers_docker.py b/neurodocker/reproenv/tests/test_renderers_docker.py index 3a2fc678..74779fed 100644 --- a/neurodocker/reproenv/tests/test_renderers_docker.py +++ b/neurodocker/reproenv/tests/test_renderers_docker.py @@ -307,13 +307,13 @@ def test_docker_render_from_instance_methods(): ) d = DockerRenderer("apt") - d.from_("debian:buster-slim") + d.from_("debian:bullseye-slim") d.entrypoint(["echo", "foo bar"]) rendered = str(d) rendered = prune_rendered(rendered).strip() assert ( rendered == """\ -FROM debian:buster-slim +FROM debian:bullseye-slim ENTRYPOINT ["echo", "foo bar"]""" ) diff --git a/neurodocker/reproenv/tests/test_renderers_singularity.py b/neurodocker/reproenv/tests/test_renderers_singularity.py index 5a4b0b32..bbf60f41 100644 --- a/neurodocker/reproenv/tests/test_renderers_singularity.py +++ b/neurodocker/reproenv/tests/test_renderers_singularity.py @@ -323,7 +323,7 @@ def test_singularity_render_from_instance_methods(): ) s = SingularityRenderer("apt") - s.from_("debian:buster-slim") + s.from_("debian:bullseye-slim") s.entrypoint(["echo", "foobar baz"]) rendered = str(s) rendered = prune_rendered(rendered).strip() @@ -331,7 +331,7 @@ def test_singularity_render_from_instance_methods(): rendered == """\ Bootstrap: docker -From: debian:buster-slim +From: debian:bullseye-slim %post diff --git a/neurodocker/reproenv/tests/utils.py b/neurodocker/reproenv/tests/utils.py index d1111150..d55c5b7c 100644 --- a/neurodocker/reproenv/tests/utils.py +++ b/neurodocker/reproenv/tests/utils.py @@ -24,7 +24,10 @@ def _docker_available(): """Return `True` if docker-py is installed or docker engine is available.""" if docker is None: return False - client = docker.from_env() + try: + client = docker.from_env() + except docker.errors.DockerException: + return False try: return client.ping() # bool, unless engine is unresponsive (eg not installed) except docker.errors.APIError: @@ -102,15 +105,18 @@ def build_singularity_image(context: Path, remove=True) -> Generator[str, None, cachedir = Path("/") / "dev" / "shm" / user / "apptainer" singularity = os.environ.get("REPROENV_APPTAINER_PROGRAM", "apptainer") cmd: list[str] = [ - "sudo", - f"APPTAINER_CACHEDIR={cachedir}", singularity, "build", + "--fakeroot", str(sif), str(recipe), ] + env: dict[str, str] = { + "APPTAINER_CACHEDIR": cachedir, + "PATH": os.environ.get("PATH"), + } try: - _ = subprocess.check_output(cmd, cwd=context) + _ = subprocess.check_output(cmd, env=env, cwd=context) yield str(sif) finally: if remove: @@ -153,7 +159,8 @@ def run_singularity_image( """ scmd = "run" if entrypoint is None else "exec" # sudo not required - cmd: list[str] = ["singularity", scmd, "--cleanenv", img] + singularity = os.environ.get("REPROENV_APPTAINER_PROGRAM", "apptainer") + cmd: list[str] = [singularity, scmd, "--cleanenv", img] if entrypoint is not None: cmd.extend(entrypoint) if args is not None: diff --git a/neurodocker/templates/_default.yaml b/neurodocker/templates/_default.yaml index 98a401ff..ee2bc03d 100644 --- a/neurodocker/templates/_default.yaml +++ b/neurodocker/templates/_default.yaml @@ -19,16 +19,20 @@ source: yum: - bzip2 - ca-certificates - - curl - epel-release - - localedef - unzip + - glibc-langpack-en + - glibc-locale-source env: LANG: en_US.UTF-8 LC_ALL: en_US.UTF-8 ND_ENTRYPOINT: /neurodocker/startup.sh instructions: | export ND_ENTRYPOINT="{{ self.env['ND_ENTRYPOINT'] }}" + {%- if self.pkg_manager == "yum" %} + sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + {%- endif %} {{ self.install_dependencies() }} {%- if self.pkg_manager == "apt" %} sed -i -e 's/# {{ self.env['LC_ALL'] }} UTF-8/{{ self.env['LC_ALL'] }} UTF-8/' /etc/locale.gen diff --git a/neurodocker/templates/jq.yaml b/neurodocker/templates/jq.yaml index 6d2d6d3b..fa5b5b30 100644 --- a/neurodocker/templates/jq.yaml +++ b/neurodocker/templates/jq.yaml @@ -18,8 +18,9 @@ binaries: curl -fsSL --output /usr/local/bin/jq {{ self.urls[self.version]}} chmod +x /usr/local/bin/jq urls: - '1.6': https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 - '1.5': https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 + 1.7.1: https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux64 + '1.7': https://github.com/jqlang/jq/releases/download/jq-1.7/jq-linux64 + '1.6': https://github.com/jqlang/jq/releases/download/jq-1.6/jq-linux64 source: arguments: required: @@ -44,9 +45,13 @@ source: instructions: | mkdir jq cd jq - curl -fsSL https://github.com/stedolan/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \ + curl -fsSL https://github.com/jqlang/jq/releases/download/jq-{{self.version}}/jq-{{self.version}}.tar.gz \ | tar xz --strip-components 1 - {% if self.version == "1.6" -%} autoreconf -fi {% endif %} - ./configure --disable-maintainer-mode + {%- if self.version == '1.6' %} + autoreconf -i + ./configure --with-oniguruma=builtin --disable-maintainer-mode + {%- else %} + ./configure --with-oniguruma=builtin + {%- endif %} make make install