Merge branch 'develop' into container_test

Author: boegel

easybuild/tools/config.py

@@ -134,7 +134,7 @@
 JOB_DEPS_TYPE_ABORT_ON_ERROR = 'abort_on_error'
 JOB_DEPS_TYPE_ALWAYS_RUN = 'always_run'
 
-DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:16.04'
+DOCKER_BASE_IMAGE_UBUNTU = 'ubuntu:20.04'
 DOCKER_BASE_IMAGE_CENTOS = 'centos:7'
 
 LOCAL_VAR_NAMING_CHECK_ERROR = 'error'

easybuild/tools/containers/docker.py

@@ -35,6 +35,7 @@
 from easybuild.tools.containers.base import ContainerGenerator
 from easybuild.tools.containers.utils import det_os_deps
 from easybuild.tools.filetools import remove_dir
+from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
 from easybuild.tools.run import run_cmd
 
 
@@ -44,9 +45,9 @@
 """
 
 DOCKER_INSTALL_EASYBUILD = """\
-RUN pip install -U pip setuptools && \\
-    hash -r pip && \\
-    pip install -U easybuild
+RUN pip3 install -U pip setuptools && \\
+    hash -r pip3&& \\
+    pip3 install -U easybuild
 
 RUN mkdir /app && \\
     mkdir /scratch && \\
@@ -61,37 +62,43 @@
 
 RUN set -x && \\
     . /usr/share/lmod/lmod/init/sh && \\
-    eb %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp
+    eb --robot %(eb_opts)s --installpath=/app/ --prefix=/scratch --tmpdir=/scratch/tmp
 
-RUN touch ${HOME}/.profile && \\
-    echo '\\n# Added by easybuild docker packaging' >> ${HOME}/.profile && \\
-    echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.profile && \\
-    echo 'module use %(init_modulepath)s' >> ${HOME}/.profile && \\
-    echo 'module load %(mod_names)s' >> ${HOME}/.profile
+RUN touch ${HOME}/.bashrc && \\
+    echo '' >> ${HOME}/.bashrc && \\
+    echo '# Added by easybuild docker packaging' >> ${HOME}/.bashrc && \\
+    echo 'source /usr/share/lmod/lmod/init/bash' >> ${HOME}/.bashrc && \\
+    echo 'module use %(init_modulepath)s' >> ${HOME}/.bashrc && \\
+    echo 'module load %(mod_names)s' >> ${HOME}/.bashrc
 
 CMD ["/bin/bash", "-l"]
 """
 
-DOCKER_UBUNTU1604_INSTALL_DEPS = """\
+DOCKER_UBUNTU2004_INSTALL_DEPS = """\
 RUN apt-get update && \\
-    apt-get install -y python python-pip lmod curl wget
+    DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip lmod \\
+        curl wget git bzip2 gzip tar zip unzip xz-utils \\
+        patch automake git debianutils \\
+        g++ libdata-dump-perl libthread-queue-any-perl libssl-dev
 
 RUN OS_DEPS='%(os_deps)s' && \\
-    test -n "${OS_DEPS}" && \\
     for dep in ${OS_DEPS}; do apt-get -qq install ${dep} || true; done
 """
 
 DOCKER_CENTOS7_INSTALL_DEPS = """\
 RUN yum install -y epel-release && \\
-    yum install -y python python-pip Lmod curl wget git
+    yum install -y python3 python3-pip Lmod curl wget git \\
+        bzip2 gzip tar zip unzip xz \\
+        patch make git which \\
+        gcc-c++ perl-Data-Dumper perl-Thread-Queue openssl-dev
 
 RUN OS_DEPS='%(os_deps)s' && \\
     test -n "${OS_DEPS}" && \\
     yum --skip-broken install -y "${OS_DEPS}" || true
 """
 
 DOCKER_OS_INSTALL_DEPS_TMPLS = {
-    DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU1604_INSTALL_DEPS,
+    DOCKER_BASE_IMAGE_UBUNTU: DOCKER_UBUNTU2004_INSTALL_DEPS,
     DOCKER_BASE_IMAGE_CENTOS: DOCKER_CENTOS7_INSTALL_DEPS,
 }
 
@@ -125,9 +132,12 @@ def resolve_template_data(self):
 
         ec = self.easyconfigs[-1]['ec']
 
-        init_modulepath = os.path.join("/app/modules/all", *self.mns.det_init_modulepaths(ec))
+        # We are using the default MNS inside the container
+        docker_mns = EasyBuildMNS()
+
+        init_modulepath = os.path.join("/app/modules/all", *docker_mns.det_init_modulepaths(ec))
 
-        mod_names = [e['ec'].full_mod_name for e in self.easyconfigs]
+        mod_names = [docker_mns.det_full_module_name(e['ec']) for e in self.easyconfigs]
 
         eb_opts = [os.path.basename(e['spec']) for e in self.easyconfigs]
 

easybuild/tools/containers/singularity.py

@@ -272,9 +272,9 @@ def resolve_template_data(self):
                 # EPEL is required for installing Lmod & python-pip
                 'epel-release',
                 # EasyBuild requirements
-                'python setuptools Lmod',
-                # pip is used to install EasyBuild packages
-                'python-pip',
+                'python3 setuptools Lmod',
+                # pip3 is used to install EasyBuild packages
+                'python3-pip',
                 # useful utilities
                 'bzip2 gzip tar zip unzip xz',  # extracting sources
                 'curl wget',  # downloading
@@ -308,13 +308,13 @@ def resolve_template_data(self):
         template_data['install_os_deps'] = '\n'.join(install_os_deps)
 
         # install (latest) EasyBuild in container image
-        # use 'pip install', unless custom commands are specified via 'install_eb' keyword
+        # use 'pip3 install', unless custom commands are specified via 'install_eb' keyword
         if 'install_eb' not in template_data:
             template_data['install_eb'] = '\n'.join([
-                "# install EasyBuild using pip",
+                "# install EasyBuild using pip3",
                 # upgrade pip
-                "pip install -U pip",
-                "pip install easybuild",
+                "pip3 install -U pip",
+                "pip3 install easybuild",
             ])
 
         # if no custom value is specified for 'post_commands' keyword,

easybuild/tools/systemtools.py

@@ -168,6 +168,7 @@
     '7z': "extracting sources (.iso)",
     'bunzip2': "decompressing sources (.bz2, .tbz, .tbz2, ...)",
     DPKG: "checking OS dependencies (Debian, Ubuntu, ...)",
+    'git': "downloading sources using 'git clone'",
     'gunzip': "decompressing source files (.gz, .tgz, ...)",
     'make': "build tool",
     'patch': "applying patch files",
@@ -741,40 +742,31 @@ def get_os_version():
                 os_version = res.group('version')
 
     if os_version:
-        if get_os_name() in ["suse", "SLES"]:
-
-            # SLES subversions can only be told apart based on kernel version,
-            # see http://wiki.novell.com/index.php/Kernel_versions
-            version_suffixes = {
-                '11': [
-                    ('2.6.27', ''),
-                    ('2.6.32', '_SP1'),
-                    ('3.0.101-63', '_SP4'),
-                    # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough?
-                    ('3.0.101', '_SP3'),
-                    # SP2 kernel versions range from 3.0.13 - 3.0.101
-                    ('3.0', '_SP2'),
-                ],
-
-                '12': [
-                    ('3.12.28', ''),
-                    ('3.12.49', '_SP1'),
-                ],
-            }
-
+        # older SLES subversions can only be told apart based on kernel version,
+        # see http://wiki.novell.com/index.php/Kernel_versions
+        sles_version_suffixes = {
+            '11': [
+                ('2.6.27', ''),
+                ('2.6.32', '_SP1'),
+                ('3.0.101-63', '_SP4'),
+                # not 100% correct, since early SP3 had 3.0.76 - 3.0.93, but close enough?
+                ('3.0.101', '_SP3'),
+                # SP2 kernel versions range from 3.0.13 - 3.0.101
+                ('3.0', '_SP2'),
+            ],
+
+            '12': [
+                ('3.12.28', ''),
+                ('3.12.49', '_SP1'),
+            ],
+        }
+        if get_os_name() in ['suse', 'SLES'] and os_version in sles_version_suffixes:
             # append suitable suffix to system version
-            if os_version in version_suffixes.keys():
-                kernel_version = platform.uname()[2]
-                known_sp = False
-                for (kver, suff) in version_suffixes[os_version]:
-                    if kernel_version.startswith(kver):
-                        os_version += suff
-                        known_sp = True
-                        break
-                if not known_sp:
-                    suff = '_UNKNOWN_SP'
-            else:
-                raise EasyBuildError("Don't know how to determine subversions for SLES %s", os_version)
+            kernel_version = platform.uname()[2]
+            for (kver, suff) in sles_version_suffixes[os_version]:
+                if kernel_version.startswith(kver):
+                    os_version += suff
+                    break
 
         return os_version
     else:

setup.py

@@ -84,7 +84,7 @@ def find_rel_test():
 implement support for installing particular (groups of) software packages.""",
     license="GPLv2",
     keywords="software build building installation installing compilation HPC scientific",
-    url="https://easybuilders.github.io/easybuild",
+    url="https://easybuild.io",
     packages=easybuild_packages,
     package_dir={'test.framework': 'test/framework'},
     package_data={'test.framework': find_rel_test()},
@@ -116,6 +116,7 @@ def find_rel_test():
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Topic :: Software Development :: Build Tools",
     ],
     platforms="Linux",

test/framework/containers.py

@@ -141,8 +141,8 @@ def test_end2end_singularity_recipe_config(self):
             self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
 
         pip_patterns = [
-            # EasyBuild is installed with pip by default
-            "pip install easybuild",
+            # EasyBuild is installed with pip3 by default
+            "pip3 install easybuild",
         ]
         post_commands_patterns = [
             # easybuild user is added if it doesn't exist yet
@@ -386,7 +386,7 @@ def test_end2end_dockerfile(self):
                               base_args + ['--container-config=not-supported'],
                               raise_error=True)
 
-        for cont_base in ['ubuntu:16.04', 'centos:7']:
+        for cont_base in ['ubuntu:20.04', 'centos:7']:
             stdout, stderr = self.run_main(base_args + ['--container-config=%s' % cont_base])
             self.assertFalse(stderr)
             regexs = ["^== Dockerfile definition file created at %s/containers/Dockerfile.toy-0.0" % self.test_prefix]
@@ -406,11 +406,11 @@ def test_end2end_dockerfile(self):
         remove_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
 
         base_args.insert(1, os.path.join(test_ecs, 'g', 'GCC', 'GCC-4.9.2.eb'))
-        self.run_main(base_args + ['--container-config=ubuntu:16.04'])
+        self.run_main(base_args + ['--container-config=ubuntu:20.04'])
         def_file = read_file(os.path.join(self.test_prefix, 'containers', 'Dockerfile.toy-0.0'))
         regexs = [
-            "FROM ubuntu:16.04",
-            "eb toy-0.0.eb GCC-4.9.2.eb",
+            "FROM ubuntu:20.04",
+            "eb --robot toy-0.0.eb GCC-4.9.2.eb",
             "module load toy/0.0 GCC/4.9.2",
         ]
         self.check_regexs(regexs, def_file)
@@ -435,7 +435,7 @@ def test_end2end_docker_image(self):
             '-C',  # equivalent with --containerize
             '--experimental',
             '--container-type=docker',
-            '--container-config=ubuntu:16.04',
+            '--container-config=ubuntu:20.04',
             '--container-build-image',
         ]
 

test/framework/systemtools.py

@@ -366,10 +366,13 @@ def setUp(self):
         self.orig_is_readable = st.is_readable
         self.orig_read_file = st.read_file
         self.orig_run_cmd = st.run_cmd
+        self.orig_platform_dist = st.platform.dist if hasattr(st.platform, 'dist') else None
         self.orig_platform_uname = st.platform.uname
         self.orig_get_tool_version = st.get_tool_version
         self.orig_sys_version_info = st.sys.version_info
         self.orig_HAVE_ARCHSPEC = st.HAVE_ARCHSPEC
+        self.orig_HAVE_DISTRO = st.HAVE_DISTRO
+        self.orig_ETC_OS_RELEASE = st.ETC_OS_RELEASE
         if hasattr(st, 'archspec_cpu_host'):
             self.orig_archspec_cpu_host = st.archspec_cpu_host
         else:
@@ -383,10 +386,14 @@ def tearDown(self):
         st.get_os_name = self.orig_get_os_name
         st.get_os_type = self.orig_get_os_type
         st.run_cmd = self.orig_run_cmd
+        if self.orig_platform_dist is not None:
+            st.platform.dist = self.orig_platform_dist
         st.platform.uname = self.orig_platform_uname
         st.get_tool_version = self.orig_get_tool_version
         st.sys.version_info = self.orig_sys_version_info
         st.HAVE_ARCHSPEC = self.orig_HAVE_ARCHSPEC
+        st.HAVE_DISTRO = self.orig_HAVE_DISTRO
+        st.ETC_OS_RELEASE = self.orig_ETC_OS_RELEASE
         if self.orig_archspec_cpu_host is not None:
             st.archspec_cpu_host = self.orig_archspec_cpu_host
         super(SystemToolsTest, self).tearDown()
@@ -738,6 +745,21 @@ def test_os_version(self):
         os_version = get_os_version()
         self.assertTrue(isinstance(os_version, string_type) or os_version == UNKNOWN)
 
+        # make sure that bug fixed in https://github.com/easybuilders/easybuild-framework/issues/3952
+        # does not surface again, by mocking what's needed to make get_os_version fall into SLES-specific path
+
+        if hasattr(st.platform, 'dist'):
+            st.platform.dist = lambda: (None, None)
+        st.HAVE_DISTRO = False
+
+        st.get_os_name = lambda: 'SLES'
+        fake_etc_os_release = os.path.join(self.test_prefix, 'os-release')
+        write_file(fake_etc_os_release, 'VERSION="15-SP1"')
+        st.ETC_OS_RELEASE = fake_etc_os_release
+
+        os_version = get_os_version()
+        self.assertEqual(os_version, '15-SP1')
+
     def test_gcc_version_native(self):
         """Test getting gcc version."""
         gcc_version = get_gcc_version()