Merge pull request #3976 from ItIsI-Orient/20220308161919_new_pr_HyjCenCVwv

Added new parameter key to specify some download or installation steps for user in case of complicated way of obtaining needed files

Author: bartoldeman

easybuild/framework/easyblock.py

@@ -364,7 +364,7 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         else:
             raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums))
 
-    def fetch_source(self, source, checksum=None, extension=False):
+    def fetch_source(self, source, checksum=None, extension=False, download_instructions=None):
         """
         Get a specific source (tarball, iso, url)
         Will be tested for existence or can be located
@@ -400,7 +400,8 @@ def fetch_source(self, source, checksum=None, extension=False):
         # check if the sources can be located
         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
         path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
-                                force_download=force_download, urls=source_urls, git_config=git_config)
+                                force_download=force_download, urls=source_urls, git_config=git_config,
+                                download_instructions=download_instructions)
         if path is None:
             raise EasyBuildError('No file found for source %s', filename)
 
@@ -586,7 +587,8 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                             source['source_urls'] = source_urls
 
                         if fetch_files:
-                            src = self.fetch_source(source, checksums, extension=True)
+                            src = self.fetch_source(source, checksums, extension=True,
+                                                    download_instructions=ext_options.get('download_instructions'))
                             ext_src.update({
                                 # keep track of custom extract command (if any)
                                 'extract_cmd': src['cmd'],
@@ -682,7 +684,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
         return exts_sources
 
     def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
-                    git_config=None):
+                    git_config=None, download_instructions=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -869,8 +871,19 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     self.dry_run_msg("  * %s (MISSING)", filename)
                     return filename
                 else:
-                    raise EasyBuildError("Couldn't find file %s anywhere, and downloading it didn't work either... "
-                                         "Paths attempted (in order): %s ", filename, ', '.join(failedpaths))
+                    error_msg = "Couldn't find file %s anywhere, "
+                    if download_instructions is None:
+                        download_instructions = self.cfg['download_instructions']
+                    if download_instructions is not None and download_instructions != "":
+                        msg = "\nDownload instructions:\n\n" + download_instructions + '\n'
+                        print_msg(msg, prefix=False, stderr=True)
+                        error_msg += "please follow the download instructions above, and make the file available "
+                        error_msg += "in the active source path (%s)" % ':'.join(source_paths())
+                    else:
+                        error_msg += "and downloading it didn't work either... "
+                        error_msg += "Paths attempted (in order): %s " % ', '.join(failedpaths)
+
+                    raise EasyBuildError(error_msg, filename)
 
     #
     # GETTER/SETTER UTILITY FUNCTIONS

easybuild/framework/easyconfig/default.py

@@ -90,6 +90,7 @@
     'checksums': [[], "Checksums for sources and patches", BUILD],
     'configopts': ['', 'Extra options passed to configure (default already has --prefix)', BUILD],
     'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with (if supported)", BUILD],
+    'download_instructions': ['', "Specify steps to aquire necessary file, if obtaining it is difficult", BUILD],
     'easyblock': [None, "EasyBlock to use for building; if set to None, an easyblock is selected "
                         "based on the software name", BUILD],
     'easybuild_version': [None, "EasyBuild-version this spec-file was written for", BUILD],

test/framework/easyblock.py

@@ -1480,6 +1480,110 @@ def test_fetch_sources(self):
         error_pattern = "Found one or more unexpected keys in 'sources' specification: {'nosuchkey': 'foobar'}"
         self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_sources, sources, checksums=[])
 
+    def test_download_instructions(self):
+        """Test use of download_instructions easyconfig parameter."""
+        orig_test_ec = '\n'.join([
+            "easyblock = 'ConfigureMake'",
+            "name = 'software_with_missing_sources'",
+            "version = '0.0'",
+            "homepage = 'https://example.com'",
+            "description = 'test'",
+            "toolchain = SYSTEM",
+            "sources = [SOURCE_TAR_GZ]",
+            "exts_list = [",
+            "    ('ext_with_missing_sources', '0.0', {",
+            "         'sources': [SOURCE_TAR_GZ],",
+            "    }),",
+            "]",
+        ])
+        self.contents = orig_test_ec
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+
+        common_error_pattern = "^Couldn't find file software_with_missing_sources-0.0.tar.gz anywhere"
+        error_pattern = common_error_pattern + ", and downloading it didn't work either"
+        self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
+
+        download_instructions = "download_instructions = 'Manual download from example.com required'"
+        sources = "sources = [SOURCE_TAR_GZ]"
+        self.contents = self.contents.replace(sources, download_instructions + '\n' + sources)
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+
+        error_pattern = common_error_pattern + ", please follow the download instructions above"
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
+        stderr = self.get_stderr().strip()
+        stdout = self.get_stdout().strip()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required")
+        self.assertEqual(stdout, '')
+
+        # create dummy source file
+        write_file(os.path.join(os.path.dirname(self.eb_file), 'software_with_missing_sources-0.0.tar.gz'), '')
+
+        # now downloading of sources for extension should fail
+        # top-level download instructions are printed (because there's nothing else)
+        error_pattern = "^Couldn't find file ext_with_missing_sources-0.0.tar.gz anywhere"
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
+        stderr = self.get_stderr().strip()
+        stdout = self.get_stdout().strip()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stderr, "Download instructions:\n\nManual download from example.com required")
+        self.assertEqual(stdout, '')
+
+        # wipe top-level download instructions, try again
+        self.contents = self.contents.replace(download_instructions, '')
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+
+        # no download instructions printed anymore now
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
+        stderr = self.get_stderr().strip()
+        stdout = self.get_stdout().strip()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stdout, '')
+
+        # inject download instructions for extension
+        download_instructions = ' ' * 8 + "'download_instructions': "
+        download_instructions += "'Extension sources must be downloaded via example.com',"
+        sources = "'sources': [SOURCE_TAR_GZ],"
+        self.contents = self.contents.replace(sources, sources + '\n' + download_instructions)
+        self.writeEC()
+        eb = EasyBlock(EasyConfig(self.eb_file))
+
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        self.assertErrorRegex(EasyBuildError, error_pattern, eb.fetch_step)
+        stderr = self.get_stderr().strip()
+        stdout = self.get_stdout().strip()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stderr, "Download instructions:\n\nExtension sources must be downloaded via example.com")
+        self.assertEqual(stdout, '')
+
+        # create dummy source file for extension
+        write_file(os.path.join(os.path.dirname(self.eb_file), 'ext_with_missing_sources-0.0.tar.gz'), '')
+
+        # no more errors, all source files found (so no download instructions printed either)
+        self.mock_stderr(True)
+        self.mock_stdout(True)
+        eb.fetch_step()
+        stderr = self.get_stderr().strip()
+        stdout = self.get_stdout().strip()
+        self.mock_stderr(False)
+        self.mock_stdout(False)
+        self.assertEqual(stderr, '')
+        self.assertEqual(stdout, '')
+
     def test_fetch_patches(self):
         """Test fetch_patches method."""
         testdir = os.path.abspath(os.path.dirname(__file__))