Merge pull request #3945 from scimerman/develop

switch to using pip3 for installing EasyBuild in Singularity definition file generated by EasyBuild

Author: boegel

easybuild/tools/config.py

@@ -134,7 +134,7 @@
 JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error'
 JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run'
 
-DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04'
+DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:20.04'
 DOCKER_BASE_IMAGE_CENTOS = 'centos:7'
 
 LOCAL_VAR_NAMING_CHECK_ERROR = 'error'

easybuild/tools/containers/docker.py

@@ -35,6 +35,7 @@
 from easybuild.tools.containers.base import ContainerGenerator
 from easybuild.tools.containers.utils import det_os_deps
 from easybuild.tools.filetools import remove_dir
+from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
 from easybuild.tools.run import run_cmd
 
 
@@ -44,9 +45,9 @@
 """
 
 DOCKER_INSTALL_EASYBUILD = """\
-RUN pip install -U pip setuptools && \\
-    hash -r pip && \\
-    pip install -U easybuild
+RUN pip3 install -U pip setuptools && \\
+    hash -r pip3&& \\
+    pip3 install -U easybuild
 
 RUN mkdir /app && \\
     mkdir /scratch && \\
@@ -61,37 +62,43 @@
 
 RUN set -x && \\
     . /usr/share/lmod/lmod/init/sh && \\
-    eb %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp
+    eb --robot %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp
 
-RUN touch ${HOME}/.profile && \\
-    echo '\\n# Added by easybuild docker packaging' >> ${HOME}/.profile && \\
-    echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.profile && \\
-    echo 'module use %(init_modulepath)s' >> ${HOME}/.profile && \\
-    echo 'module load %(mod_names)s' >> ${HOME}/.profile
+RUN touch ${HOME}/.bashrc && \\
+    echo '' >> ${HOME}/.bashrc && \\
+    echo '# Added by easybuild docker packaging' >> ${HOME}/.bashrc && \\
+    echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.bashrc && \\
+    echo 'module use %(init_modulepath)s' >> ${HOME}/.bashrc && \\
+    echo 'module load %(mod_names)s' >> ${HOME}/.bashrc
 
 CMD ["/bin/bash", "-l"]
 """
 
-DOCKER_UBUNTU1604_INSTALL_DEPS = """\
+DOCKER_UBUNTU2004_INSTALL_DEPS = """\
 RUN apt-get update && \\
-    apt-get install -y python python-pip lmod curl wget
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip lmod \\
+        curl wget git bzip2 gzip tar zip unzip xz-utils \\
+        patch automake git debianutils \\
+        g++ libdata-dump-perl libthread-queue-any-perl libssl-dev
 
 RUN OS_DEPS='%(os_deps)s' && \\
-    test -n "${OS_DEPS}" && \\
     for dep in ${OS_DEPS}; do apt-get -qq install ${dep} || true; done
 """
 
 DOCKER_CENTOS7_INSTALL_DEPS = """\
 RUN yum install -y epel-release && \\
-    yum install -y python python-pip Lmod curl wget git
+    yum install -y python3 python3-pip Lmod curl wget git \\
+        bzip2 gzip tar zip unzip xz \\
+        patch make git which \\
+        gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev
 
 RUN OS_DEPS='%(os_deps)s' && \\
     test -n "${OS_DEPS}" && \\
     yum --skip-broken install -y "${OS_DEPS}" || true
 """
 
 DOCKER_OS_INSTALL_DEPS_TMPLS = {
-    DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU1604_INSTALL_DEPS,
+    DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU2004_INSTALL_DEPS,
     DOCKER_BASE_IMAGE_CENTOS: DOCKER_CENTOS7_INSTALL_DEPS,
 }
 
@@ -125,9 +132,12 @@ def resolve_template_data(self):
 
         ec = self.easyconfigs[-1]['ec']
 
-        init_modulepath = os.path.join("/app/modules/all", *self.mns.det_init_modulepaths(ec))
+        # We are using the default MNS inside the container
+        docker_mns = EasyBuildMNS()
+
+        init_modulepath = os.path.join("/app/modules/all", *docker_mns.det_init_modulepaths(ec))
 
-        mod_names = [e['ec'].full_mod_name for e in self.easyconfigs]
+        mod_names = [docker_mns.det_full_module_name(e['ec']) for e in self.easyconfigs]
 
         eb_opts = [os.path.basename(e['spec']) for e in self.easyconfigs]
 

easybuild/tools/containers/singularity.py

@@ -272,9 +272,9 @@ def resolve_template_data(self):
                 # EPEL is required for installing Lmod & python-pip
                 'epel-release',
                 # EasyBuild requirements
-                'python setuptools Lmod',
-                # pip is used to install EasyBuild packages
-                'python-pip',
+                'python3 setuptools Lmod',
+                # pip3 is used to install EasyBuild packages
+                'python3-pip',
                 # useful utilities
                 'bzip2 gzip tar zip unzip xz',  # extracting sources
                 'curl wget',  # downloading
@@ -308,13 +308,13 @@ def resolve_template_data(self):
         template_data['install_os_deps'] = '\n'.join(install_os_deps)
 
         # install (latest) EasyBuild in container image
-        # use 'pip install', unless custom commands are specified via 'install_eb' keyword
+        # use 'pip3 install', unless custom commands are specified via 'install_eb' keyword
         if 'install_eb' not in template_data:
             template_data['install_eb'] = '\n'.join([
-                "# install EasyBuild using pip",
+                "# install EasyBuild using pip3",
                 # upgrade pip
-                "pip install -U pip",
-                "pip install easybuild",
+                "pip3 install -U pip",
+                "pip3 install easybuild",
             ])
 
         # if no custom value is specified for 'post_commands' keyword,

test/framework/containers.py

@@ -141,8 +141,8 @@ def test_end2end_singularity_recipe_config(self):
             self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
 
         pip_patterns = [
-            # EasyBuild is installed with pip by default
-            "pip install easybuild",
+            # EasyBuild is installed with pip3 by default
+            "pip3 install easybuild",
         ]
         post_commands_patterns = [
             # easybuild user is added if it doesn't exist yet
@@ -386,7 +386,7 @@ def test_end2end_dockerfile(self):
                               base_args + ['--container-config=not-supported'],
                               raise_error=True)
 
-        for cont_base in ['ubuntu:16.04', 'centos:7']:
+        for cont_base in ['ubuntu:20.04', 'centos:7']:
             stdout, stderr = self.run_main(base_args + ['--container-config=%s' % cont_base])
             self.assertFalse(stderr)
             regexs = ["^== Dockerfile definition file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix]
@@ -406,11 +406,11 @@ def test_end2end_dockerfile(self):
         remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
 
         base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb'))
-        self.run_main(base_args + ['--container-config=ubuntu:16.04'])
+        self.run_main(base_args + ['--container-config=ubuntu:20.04'])
         def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
         regexs = [
-            "FROM ubuntu:16.04",
-            "eb toy-0.0.eb GCC-4.9.2.eb",
+            "FROM ubuntu:20.04",
+            "eb --robot toy-0.0.eb GCC-4.9.2.eb",
             "module load toy/0.0 GCC/4.9.2",
         ]
         self.check_regexs(regexs, def_file)
@@ -435,7 +435,7 @@ def test_end2end_docker_image(self):
             '-C',  # equivalent with --containerize
             '--experimental',
             '--container-type=docker',
-            '--container-config=ubuntu:16.04',
+            '--container-config=ubuntu:20.04',
             '--container-build-image',
         ]