Merge pull request #4527 from lexming/5.0.x-sync

sync with develop (20240506)

Author: boegel

easybuild/framework/easyblock.py

@@ -2351,7 +2351,7 @@ def check_readiness_step(self):
                 self.log.info("No module %s found. Not skipping anything." % self.full_mod_name)
 
         # remove existing module file under --force (but only if --skip is not used)
-        elif build_option('force') or build_option('rebuild'):
+        elif (build_option('force') or build_option('rebuild')) and not build_option('dump_env_script'):
             self.remove_module_file()
 
     def fetch_step(self, skip_checksums=False):
@@ -2609,7 +2609,7 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
 
             self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s",
-                           srcind, level, srcpathsuffix, copy)
+                           srcind, level, srcpathsuffix, copy_patch)
 
             if beginpath is None:
                 try:

easybuild/framework/extensioneasyblock.py

@@ -126,18 +126,23 @@ def _set_start_dir(self):
         elif ext_start_dir is None:
             # This may be on purpose, e.g. for Python WHL files which do not get extracted
             self.log.debug("Start dir is not set.")
-        else:
+        elif self.start_dir:
             # non-existing start dir means wrong input from user
-            warn_msg = "Provided start dir (%s) for extension %s does not exist: %s" % (self.start_dir, self.name,
-                                                                                        ext_start_dir)
+            raise EasyBuildError("Provided start dir (%s) for extension %s does not exist: %s",
+                                 self.start_dir, self.name, ext_start_dir)
+        else:
+            warn_msg = 'Failed to determine start dir for extension %s: %s' % (self.name, ext_start_dir)
             self.log.warning(warn_msg)
             print_warning(warn_msg, silent=build_option('silent'))
 
     def install_extension(self, unpack_src=False):
         """Common operations for extensions: unpacking sources, patching, ..."""
 
         # unpack file if desired
-        if unpack_src:
+        if self.options.get('nosource', False):
+            # If no source wanted use the start_dir from the main EC
+            self.ext_dir = self.master.start_dir
+        elif unpack_src:
             targetdir = os.path.join(self.master.builddir, remove_unwanted_chars(self.name))
             self.ext_dir = extract_file(self.src, targetdir, extra_options=self.unpack_options,
                                         change_into_dir=False, cmd=self.src_extract_cmd)
@@ -146,10 +151,9 @@ def install_extension(self, unpack_src=False):
             # because start_dir value is usually a relative path (if it is set)
             change_dir(self.ext_dir)
 
-            self._set_start_dir()
+        self._set_start_dir()
+        if self.start_dir:
             change_dir(self.start_dir)
-        else:
-            self._set_start_dir()
 
         # patch if needed
         EasyBlock.patch_step(self, beginpath=self.ext_dir)

easybuild/main.py

@@ -439,9 +439,9 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session
     dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
 
     keep_available_modules = any((
-        forced, dry_run_mode, options.extended_dry_run, any_pr_option_set, options.copy_ec, options.inject_checksums,
-        options.sanity_check_only, options.inject_checksums_to_json)
-    )
+        forced, dry_run_mode, any_pr_option_set, options.copy_ec, options.dump_env_script, options.extended_dry_run,
+        options.inject_checksums, options.inject_checksums_to_json, options.sanity_check_only
+    ))
 
     # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
     if not keep_available_modules:

easybuild/tools/config.py

@@ -277,6 +277,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'debug',
         'debug_lmod',
         'dump_autopep8',
+        'dump_env_script',
         'enforce_checksums',
         'experimental',
         'extended_dry_run',

test/framework/easyblock.py

@@ -1746,7 +1746,7 @@ def test_fetch_patches(self):
         self.assertEqual(eb.patches[1]['level'], 4)
         self.assertEqual(eb.patches[2]['name'], toy_patch)
         self.assertEqual(eb.patches[2]['sourcepath'], 'foobar')
-        self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz'),
+        self.assertEqual(eb.patches[3]['name'], 'toy-0.0.tar.gz')
         self.assertEqual(eb.patches[3]['copy'], 'some/path')
         self.assertEqual(eb.patches[4]['name'], toy_patch)
         self.assertEqual(eb.patches[4]['level'], 0)
@@ -2280,18 +2280,25 @@ def test_extension_set_start_dir(self):
         cwd = os.getcwd()
         self.assertExists(cwd)
 
-        def check_ext_start_dir(expected_start_dir, unpack_src=True):
+        def check_ext_start_dir(expected_start_dir, unpack_src=True, parent_startdir=None):
             """Check start dir."""
             # make sure we're in an existing directory at the start
             change_dir(cwd)
+
             eb = EasyBlock(ec['ec'])
+            if not os.path.exists(eb.builddir):
+                eb.make_builddir()  # Required to exist for samefile
+            eb.cfg['start_dir'] = parent_startdir
+
             eb.extensions_step(fetch=True, install=False)
             # extract sources of the extension
             ext = eb.ext_instances[-1]
             ext.install_extension(unpack_src=unpack_src)
 
             if expected_start_dir is None:
                 self.assertIsNone(ext.start_dir)
+                # Without a start dir we don't change the CWD
+                self.assertEqual(os.getcwd(), cwd)
             else:
                 self.assertTrue(os.path.isabs(ext.start_dir))
                 if ext.start_dir != os.sep:
@@ -2301,14 +2308,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True):
                 else:
                     abs_expected_start_dir = os.path.join(eb.builddir, expected_start_dir)
                 self.assertEqual(ext.start_dir, abs_expected_start_dir)
-                if not os.path.exists(eb.builddir):
-                    eb.make_builddir()  # Required to exist for samefile
                 self.assertTrue(os.path.samefile(ext.start_dir, abs_expected_start_dir))
-            if unpack_src:
                 self.assertTrue(os.path.samefile(os.getcwd(), abs_expected_start_dir))
-            else:
-                # When not unpacking we don't change the CWD
-                self.assertEqual(os.getcwd(), cwd)
             remove_dir(eb.builddir)
 
         ec['ec']['exts_defaultclass'] = 'DummyExtension'
@@ -2337,11 +2338,8 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True):
                 'start_dir': 'nonexistingdir'}),
         ]
         with self.mocked_stdout_stderr():
-            err_pattern = "Failed to change from .*barbar/barbar-0.0 to nonexistingdir.*"
+            err_pattern = r"Provided start dir \(nonexistingdir\) for extension barbar does not exist:.*"
             self.assertErrorRegex(EasyBuildError, err_pattern, check_ext_start_dir, 'whatever')
-            stderr = self.get_stderr()
-        warning_pattern = "WARNING: Provided start dir (nonexistingdir) for extension barbar does not exist"
-        self.assertIn(warning_pattern, stderr)
 
         # No error when using relative path in non-extracted source for some reason
         ec['ec']['exts_list'] = [
@@ -2371,6 +2369,15 @@ def check_ext_start_dir(expected_start_dir, unpack_src=True):
             check_ext_start_dir(os.sep, unpack_src=False)
             self.assertFalse(self.get_stderr())
 
+        # Go to ECs start dir if nosource is used
+        ec['ec']['exts_list'] = [
+            ('barbar', '0.0', {
+                'nosource': True}),
+        ]
+        with self.mocked_stdout_stderr():
+            check_ext_start_dir(self.test_prefix, parent_startdir=self.test_prefix)
+            self.assertFalse(self.get_stderr())
+
     def test_prepare_step(self):
         """Test prepare step (setting up build environment)."""
         test_easyconfigs = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs')

test/framework/options.py

@@ -4630,7 +4630,7 @@ def test_github_new_update_pr(self):
         res = [d for d in res if os.path.basename(d) != os.path.basename(git_working_dir)]
         if len(res) == 1:
             unstaged_file_full = os.path.join(res[0], unstaged_file)
-            self.assertNotExists(unstaged_file_full), "%s not found in %s" % (unstaged_file, res[0])
+            self.assertNotExists(unstaged_file_full)
         else:
             self.fail("Found copy of easybuild-easyconfigs working copy")
 
@@ -5361,6 +5361,45 @@ def test_dump_env_script(self):
         ])
         self.assertEqual(res.output.strip(), expected_out)
 
+    def test_dump_env_script_existing_module(self):
+        toy_ec = 'toy-0.0.eb'
+
+        os.chdir(self.test_prefix)
+        self._run_mock_eb([toy_ec, '--force'], do_build=True)
+        env_script = os.path.join(self.test_prefix, os.path.splitext(toy_ec)[0] + '.env')
+        test_module = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+        if get_module_syntax() == 'Lua':
+            test_module += '.lua'
+        self.assertExists(test_module)
+        self.assertNotExists(env_script)
+
+        args = [toy_ec, '--dump-env']
+        os.chdir(self.test_prefix)
+        self._run_mock_eb(args, do_build=True, raise_error=True)
+        self.assertExists(env_script)
+        self.assertExists(test_module)
+        module_content = read_file(test_module)
+        env_file_content = read_file(env_script)
+
+        error_msg = (r"Script\(s\) already exists, not overwriting them \(unless --force is used\): "
+                     + os.path.basename(env_script))
+        os.chdir(self.test_prefix)
+        self.assertErrorRegex(EasyBuildError, error_msg, self._run_mock_eb, args, do_build=True, raise_error=True)
+        self.assertExists(env_script)
+        self.assertExists(test_module)
+        # Unchanged module and env file
+        self.assertEqual(read_file(test_module), module_content)
+        self.assertEqual(read_file(env_script), env_file_content)
+
+        args.append('--force')
+        os.chdir(self.test_prefix)
+        self._run_mock_eb(args, do_build=True, raise_error=True)
+        self.assertExists(env_script)
+        self.assertExists(test_module)
+        # Unchanged module and env file
+        self.assertEqual(read_file(test_module), module_content)
+        self.assertEqual(read_file(env_script), env_file_content)
+
     def test_stop(self):
         """Test use of --stop."""
         args = ['toy-0.0.eb', '--force', '--stop=configure']