Merge branch '5.0.x' of github.com:easybuilders/easybuild-framework into 5.0.x_fix_ci_apptainer

Author: boegel

easybuild/framework/easyblock.py

@@ -54,6 +54,7 @@
 import traceback
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
+from textwrap import indent
 
 import easybuild.tools.environment as env
 import easybuild.tools.toolchain as toolchain
@@ -953,7 +954,8 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     if download_instructions is None:
                         download_instructions = self.cfg['download_instructions']
                     if download_instructions is not None and download_instructions != "":
-                        msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
+                        msg = "\nDownload instructions:\n\n" + indent(download_instructions, '    ') + '\n\n'
+                        msg += "Make the files available in the active source path: %s\n" % ':'.join(source_paths())
                         print_msg(msg, prefix=False, stderr=True)
                         error_msg += "please follow the download instructions above, and make the file available "
                         error_msg += "in the active source path (%s)" % ':'.join(source_paths())

easybuild/framework/easyconfig/templates.py

@@ -97,6 +97,9 @@
     ('cuda_cc_cmake', "List of CUDA compute capabilities suitable for use with $CUDAARCHS in CMake 3.18+"),
     ('cuda_cc_space_sep', "Space-separated list of CUDA compute capabilities"),
     ('cuda_cc_semicolon_sep', "Semicolon-separated list of CUDA compute capabilities"),
+    ('cuda_int_comma_sep', "Comma-separated list of integer CUDA compute capabilities"),
+    ('cuda_int_space_sep', "Space-separated list of integer CUDA compute capabilities"),
+    ('cuda_int_semicolon_sep', "Semicolon-separated list of integer CUDA compute capabilities"),
     ('cuda_sm_comma_sep', "Comma-separated list of sm_* values that correspond with CUDA compute capabilities"),
     ('cuda_sm_space_sep', "Space-separated list of sm_* values that correspond with CUDA compute capabilities"),
 ]
@@ -365,6 +368,10 @@ def template_constant_dict(config, ignore=None, toolchain=None):
         template_values['cuda_cc_space_sep'] = ' '.join(cuda_compute_capabilities)
         template_values['cuda_cc_semicolon_sep'] = ';'.join(cuda_compute_capabilities)
         template_values['cuda_cc_cmake'] = ';'.join(cc.replace('.', '') for cc in cuda_compute_capabilities)
+        int_values = [cc.replace('.', '') for cc in cuda_compute_capabilities]
+        template_values['cuda_int_comma_sep'] = ','.join(int_values)
+        template_values['cuda_int_space_sep'] = ' '.join(int_values)
+        template_values['cuda_int_semicolon_sep'] = ';'.join(int_values)
         sm_values = ['sm_' + cc.replace('.', '') for cc in cuda_compute_capabilities]
         template_values['cuda_sm_comma_sep'] = ','.join(sm_values)
         template_values['cuda_sm_space_sep'] = ' '.join(sm_values)

easybuild/tools/run.py

@@ -42,6 +42,7 @@
 import re
 import signal
 import shutil
+import string
 import subprocess
 import sys
 import tempfile
@@ -187,6 +188,19 @@ def cache_aware_func(cmd, *args, **kwargs):
 run_shell_cmd_cache = run_cmd_cache
 
 
+def fileprefix_from_cmd(cmd, allowed_chars=False):
+    """
+    Simplify the cmd to only the allowed_chars we want in a filename
+
+    :param cmd: the cmd (string)
+    :param allowed_chars: characters allowed in filename (defaults to string.ascii_letters + string.digits + "_-")
+    """
+    if not allowed_chars:
+        allowed_chars = f"{string.ascii_letters}{string.digits}_-"
+
+    return ''.join([c for c in cmd if c in allowed_chars])
+
+
 @run_shell_cmd_cache
 def run_shell_cmd(cmd, fail_on_error=True, split_stderr=False, stdin=None, env=None,
                   hidden=False, in_dry_run=False, verbose_dry_run=False, work_dir=None, use_bash=True,
@@ -238,7 +252,6 @@ def to_cmd_str(cmd):
         work_dir = os.getcwd()
 
     cmd_str = to_cmd_str(cmd)
-    cmd_name = os.path.basename(cmd_str.split(' ')[0])
 
     thread_id = None
     if asynchronous:
@@ -254,6 +267,7 @@ def to_cmd_str(cmd):
     if output_file:
         toptmpdir = os.path.join(tempfile.gettempdir(), 'run-shell-cmd-output')
         os.makedirs(toptmpdir, exist_ok=True)
+        cmd_name = fileprefix_from_cmd(os.path.basename(cmd_str.split(' ')[0]))
         tmpdir = tempfile.mkdtemp(dir=toptmpdir, prefix=f'{cmd_name}-')
         cmd_out_fp = os.path.join(tmpdir, 'out.txt')
         _log.info(f'run_cmd: Output of "{cmd_str}" will be logged to {cmd_out_fp}')

test/framework/easyblock.py

@@ -1582,7 +1582,8 @@ def test_download_instructions(self):
         self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
         stderr = self.get_stderr().strip()
         self.mock_stderr(False)
-        self.assertIn("Download instructions:\n\nManual download from example.com required", stderr)
+        self.assertIn("Download instructions:\n\n    Manual download from example.com required", stderr)
+        self.assertIn("Make the files available in the active source path", stderr)
 
         # create dummy source file
         write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '')
@@ -1596,7 +1597,8 @@ def test_download_instructions(self):
         stderr = self.get_stderr().strip()
         self.mock_stderr(False)
         self.mock_stdout(False)
-        self.assertIn("Download instructions:\n\nManual download from example.com required", stderr)
+        self.assertIn("Download instructions:\n\n    Manual download from example.com required", stderr)
+        self.assertIn("Make the files available in the active source path", stderr)
 
         # wipe top-level download instructions, try again
         self.contents = self.contents.replace(download_instructions, '')
@@ -1625,7 +1627,8 @@ def test_download_instructions(self):
         stderr = self.get_stderr().strip()
         self.mock_stderr(False)
         self.mock_stdout(False)
-        self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr)
+        self.assertIn("Download instructions:\n\n    Extension sources must be downloaded via example.com", stderr)
+        self.assertIn("Make the files available in the active source path", stderr)
 
         # download instructions should also be printed if 'source_tmpl' is used to specify extension sources
         self.contents = self.contents.replace(sources, "'source_tmpl': SOURCE_TAR_GZ,")
@@ -1638,7 +1641,8 @@ def test_download_instructions(self):
         stderr = self.get_stderr().strip()
         self.mock_stderr(False)
         self.mock_stdout(False)
-        self.assertIn("Download instructions:\n\nExtension sources must be downloaded via example.com", stderr)
+        self.assertIn("Download instructions:\n\n    Extension sources must be downloaded via example.com", stderr)
+        self.assertIn("Make the files available in the active source path", stderr)
 
         # create dummy source file for extension
         write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '')

test/framework/easyconfig.py

@@ -4438,31 +4438,39 @@ def test_cuda_compute_capabilities(self):
             description = 'test'
             toolchain = SYSTEM
             cuda_compute_capabilities = ['5.1', '7.0', '7.1']
-            installopts = '%(cuda_compute_capabilities)s'
-            preinstallopts = '%(cuda_cc_space_sep)s'
-            prebuildopts = '%(cuda_cc_semicolon_sep)s'
-            configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
             preconfigopts = 'CUDAARCHS="%(cuda_cc_cmake)s"'
+            configopts = 'comma="%(cuda_sm_comma_sep)s" space="%(cuda_sm_space_sep)s"'
+            prebuildopts = '%(cuda_cc_semicolon_sep)s'
+            buildopts = ('comma="%(cuda_int_comma_sep)s" space="%(cuda_int_space_sep)s" '
+                         'semi="%(cuda_int_semicolon_sep)s"')
+            preinstallopts = '%(cuda_cc_space_sep)s'
+            installopts = '%(cuda_compute_capabilities)s'
         """)
         self.prep()
 
         ec = EasyConfig(self.eb_file)
-        self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
-        self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
-        self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
+        self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"')
         self.assertEqual(ec['configopts'], 'comma="sm_51,sm_70,sm_71" '
                                            'space="sm_51 sm_70 sm_71"')
-        self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="51;70;71"')
+        self.assertEqual(ec['prebuildopts'], '5.1;7.0;7.1')
+        self.assertEqual(ec['buildopts'], 'comma="51,70,71" '
+                                          'space="51 70 71" '
+                                          'semi="51;70;71"')
+        self.assertEqual(ec['preinstallopts'], '5.1 7.0 7.1')
+        self.assertEqual(ec['installopts'], '5.1,7.0,7.1')
 
         # build options overwrite it
         init_config(build_options={'cuda_compute_capabilities': ['4.2', '6.3']})
         ec = EasyConfig(self.eb_file)
-        self.assertEqual(ec['installopts'], '4.2,6.3')
-        self.assertEqual(ec['preinstallopts'], '4.2 6.3')
-        self.assertEqual(ec['prebuildopts'], '4.2;6.3')
+        self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"')
         self.assertEqual(ec['configopts'], 'comma="sm_42,sm_63" '
                                            'space="sm_42 sm_63"')
-        self.assertEqual(ec['preconfigopts'], 'CUDAARCHS="42;63"')
+        self.assertEqual(ec['buildopts'], 'comma="42,63" '
+                                          'space="42 63" '
+                                          'semi="42;63"')
+        self.assertEqual(ec['prebuildopts'], '4.2;6.3')
+        self.assertEqual(ec['preinstallopts'], '4.2 6.3')
+        self.assertEqual(ec['installopts'], '4.2,6.3')
 
     def test_det_copy_ec_specs(self):
         """Test det_copy_ec_specs function."""
@@ -4725,6 +4733,9 @@ def test_get_cuda_cc_template_value(self):
             'cuda_compute_capabilities': '6.5,7.0',
             'cuda_cc_space_sep': '6.5 7.0',
             'cuda_cc_semicolon_sep': '6.5;7.0',
+            'cuda_int_comma_sep': '65,70',
+            'cuda_int_space_sep': '65 70',
+            'cuda_int_semicolon_sep': '65;70',
             'cuda_sm_comma_sep': 'sm_65,sm_70',
             'cuda_sm_space_sep': 'sm_65 sm_70',
         }

test/framework/run.py

@@ -35,6 +35,7 @@
 import os
 import re
 import signal
+import string
 import stat
 import subprocess
 import sys
@@ -52,7 +53,7 @@
 from easybuild.tools.config import update_build_option
 from easybuild.tools.filetools import adjust_permissions, change_dir, mkdir, read_file, write_file
 from easybuild.tools.run import RunShellCmdResult, RunShellCmdError, check_async_cmd, check_log_for_errors
-from easybuild.tools.run import complete_cmd, get_output_from_process, parse_log_for_error
+from easybuild.tools.run import complete_cmd, fileprefix_from_cmd, get_output_from_process, parse_log_for_error
 from easybuild.tools.run import run_cmd, run_cmd_qa, run_shell_cmd, subprocess_terminate
 from easybuild.tools.config import ERROR, IGNORE, WARN
 
@@ -194,6 +195,26 @@ def test_run_shell_cmd_basic(self):
             self.assertTrue(isinstance(res.output, str))
             self.assertTrue(res.work_dir and isinstance(res.work_dir, str))
 
+    def test_fileprefix_from_cmd(self):
+        """test simplifications from fileprefix_from_cmd."""
+        cmds = {
+            'abd123': 'abd123',
+            'ab"a': 'aba',
+            'a{:$:S@"a': 'aSa',
+            'cmd-with-dash': 'cmd-with-dash',
+            'cmd_with_underscore': 'cmd_with_underscore',
+        }
+        for cmd, expected_simplification in cmds.items():
+            self.assertEqual(fileprefix_from_cmd(cmd), expected_simplification)
+
+        cmds = {
+            'abd123': 'abd',
+            'ab"a': 'aba',
+            '0a{:$:2@"a': 'aa',
+        }
+        for cmd, expected_simplification in cmds.items():
+            self.assertEqual(fileprefix_from_cmd(cmd, allowed_chars=string.ascii_letters), expected_simplification)
+
     def test_run_cmd_log(self):
         """Test logging of executed commands."""
         fd, logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')