Merge pull request #3294 from kelseymh/3290_git_in_exts-list

Add 'git_config' support within 'exts_list'

Author: boegel

easybuild/framework/easyblock.py

@@ -323,9 +323,13 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         Obtain checksum for given filename.
 
         :param checksums: a list or tuple of checksums (or None)
-        :param filename: name of the file to obtain checksum for
+        :param filename: name of the file to obtain checksum for (Deprecated)
         :param index: index of file in list
         """
+        # Filename has never been used; flag it as deprecated
+        if filename:
+            self.log.deprecated("Filename argument to get_checksum_for() is deprecated", '5.0')
+
         # if checksums are provided as a dict, lookup by source filename as key
         if isinstance(checksums, (list, tuple)):
             if index is not None and index < len(checksums) and (index >= 0 or abs(index) <= len(checksums)):
@@ -337,6 +341,59 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         else:
             raise EasyBuildError("Invalid type for checksums (%s), should be list, tuple or None.", type(checksums))
 
+    def fetch_source(self, source, checksum=None, extension=False):
+        """
+        Get a specific source (tarball, iso, url)
+        Will be tested for existence or can be located
+
+        :param source: source to be found (single dictionary in 'sources' list, or filename)
+        :param checksum: checksum corresponding to source
+        :param extension: flag if being called from fetch_extension_sources()
+        """
+        filename, download_filename, extract_cmd, source_urls, git_config = None, None, None, None, None
+
+        if source is None:
+            raise EasyBuildError("fetch_source called with empty 'source' argument")
+        elif isinstance(source, string_type):
+            filename = source
+        elif isinstance(source, dict):
+            # Making a copy to avoid modifying the object with pops
+            source = source.copy()
+            filename = source.pop('filename', None)
+            extract_cmd = source.pop('extract_cmd', None)
+            download_filename = source.pop('download_filename', None)
+            source_urls = source.pop('source_urls', None)
+            git_config = source.pop('git_config', None)
+            if source:
+                raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source)
+
+        elif isinstance(source, (list, tuple)) and len(source) == 2:
+            self.log.deprecated("Using a 2-element list/tuple to specify sources is deprecated, "
+                                "use a dictionary with 'filename', 'extract_cmd' keys instead", '4.0')
+            filename, extract_cmd = source
+        else:
+            raise EasyBuildError("Unexpected source spec, not a string or dict: %s", source)
+
+        # check if the sources can be located
+        force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+        path = self.obtain_file(filename, extension=extension, download_filename=download_filename,
+                                force_download=force_download, urls=source_urls, git_config=git_config)
+        if path is None:
+            raise EasyBuildError('No file found for source %s', filename)
+
+        self.log.debug('File %s found for source %s' % (path, filename))
+
+        src = {
+            'name': filename,
+            'path': path,
+            'cmd': extract_cmd,
+            'checksum': checksum,
+            # always set a finalpath
+            'finalpath': self.builddir,
+        }
+
+        return src
+
     def fetch_sources(self, sources=None, checksums=None):
         """
         Add a list of source files (can be tarballs, isos, urls).
@@ -350,46 +407,22 @@ def fetch_sources(self, sources=None, checksums=None):
         if checksums is None:
             checksums = self.cfg['checksums']
 
+        # Single source should be re-wrapped as a list, and checksums with it
+        if isinstance(sources, dict):
+            sources = [sources]
+        if isinstance(checksums, string_type):
+            checksums = [checksums]
+
+        # Loop over the list of sources; list of checksums must match >= in size
         for index, source in enumerate(sources):
-            extract_cmd, download_filename, source_urls, git_config = None, None, None, None
-
-            if isinstance(source, string_type):
-                filename = source
-
-            elif isinstance(source, dict):
-                # Making a copy to avoid modifying the object with pops
-                source = source.copy()
-                filename = source.pop('filename', None)
-                extract_cmd = source.pop('extract_cmd', None)
-                download_filename = source.pop('download_filename', None)
-                source_urls = source.pop('source_urls', None)
-                git_config = source.pop('git_config', None)
-                if source:
-                    raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source)
-
-            elif isinstance(source, (list, tuple)) and len(source) == 2:
-                self.log.deprecated("Using a 2-element list/tuple to specify sources is deprecated, "
-                                    "use a dictionary with 'filename', 'extract_cmd' keys instead", '4.0')
-                filename, extract_cmd = source
-            else:
-                raise EasyBuildError("Unexpected source spec, not a string or dict: %s", source)
+            if source is None:
+                raise EasyBuildError("Empty source in sources list at index %d", index)
 
-            # check if the sources can be located
-            force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
-            path = self.obtain_file(filename, download_filename=download_filename, force_download=force_download,
-                                    urls=source_urls, git_config=git_config)
-            if path:
-                self.log.debug('File %s found for source %s' % (path, filename))
-                self.src.append({
-                    'name': filename,
-                    'path': path,
-                    'cmd': extract_cmd,
-                    'checksum': self.get_checksum_for(checksums, filename=filename, index=index),
-                    # always set a finalpath
-                    'finalpath': self.builddir,
-                })
+            src_spec = self.fetch_source(source, self.get_checksum_for(checksums=checksums, index=index))
+            if src_spec:
+                self.src.append(src_spec)
             else:
-                raise EasyBuildError('No file found for source %s', filename)
+                raise EasyBuildError("Unable to retrieve source %s", source)
 
         self.log.info("Added sources: %s", self.src)
 
@@ -436,7 +469,7 @@ def fetch_patches(self, patch_specs=None, extension=False, checksums=None):
                 patchspec = {
                     'name': patch_file,
                     'path': path,
-                    'checksum': self.get_checksum_for(checksums, filename=patch_file, index=index),
+                    'checksum': self.get_checksum_for(checksums, index=index),
                 }
                 if suff:
                     if copy_file:
@@ -514,9 +547,27 @@ def fetch_extension_sources(self, skip_checksums=False):
 
                     if ext_options.get('nosource', None):
                         exts_sources.append(ext_src)
+
+                    elif ext_options.get('sources', None):
+                        sources = ext_options['sources']
+
+                        if isinstance(sources, list):
+                            if len(sources) == 1:
+                                source = sources[0]
+                            else:
+                                error_msg = "'sources' spec for %s in exts_list must be single element list"
+                                raise EasyBuildError(error_msg, ext_name, sources)
+                        else:
+                            source = sources
+
+                        src = self.fetch_source(source, checksums, extension=True)
+                        # Copy 'path' entry to 'src' for use with extensions
+                        ext_src.update({'src': src['path']})
+                        exts_sources.append(ext_src)
                     else:
                         source_urls = ext_options.get('source_urls', [])
                         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
+
                         src_fn = self.obtain_file(fn, extension=True, urls=source_urls, force_download=force_download)
 
                         if src_fn:
@@ -530,7 +581,7 @@ def fetch_extension_sources(self, skip_checksums=False):
 
                                 # verify checksum (if provided)
                                 self.log.debug('Verifying checksums for extension source...')
-                                fn_checksum = self.get_checksum_for(checksums, filename=src_fn, index=0)
+                                fn_checksum = self.get_checksum_for(checksums, index=0)
                                 if verify_checksum(src_fn, fn_checksum):
                                     self.log.info('Checksum for extension source %s verified', fn)
                                 elif build_option('ignore_checksums'):
@@ -556,7 +607,7 @@ def fetch_extension_sources(self, skip_checksums=False):
                                     self.log.debug('Verifying checksums for extension patches...')
                                     for idx, patch in enumerate(ext_patches):
                                         patch = patch['path']
-                                        checksum = self.get_checksum_for(checksums[1:], filename=patch, index=idx)
+                                        checksum = self.get_checksum_for(checksums[1:], index=idx)
                                         if verify_checksum(patch, checksum):
                                             self.log.info('Checksum for extension patch %s verified', patch)
                                         elif build_option('ignore_checksums'):

test/framework/toy_build.py

@@ -1196,6 +1196,82 @@ def test_toy_extension_patches(self):
 
         self.test_toy_build(ec_file=test_ec)
 
+    def test_toy_extension_sources_single_item_list(self):
+        """Test install toy that includes extensions with 'sources' spec (as single-item list)."""
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec)
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+
+        # test use of single-element list in 'sources' with just the filename
+        test_ec_txt = '\n'.join([
+            toy_ec_txt,
+            'exts_list = [',
+            '   ("bar", "0.0", {',
+            '       "sources": ["bar-%(version)s.tar.gz"],',
+            '   }),',
+            ']',
+        ])
+        write_file(test_ec, test_ec_txt)
+        self.test_toy_build(ec_file=test_ec)
+
+    def test_toy_extension_sources_str(self):
+        """Test install toy that includes extensions with 'sources' spec (as string value)."""
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec)
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+
+        # test use of single-element list in 'sources' with just the filename
+        test_ec_txt = '\n'.join([
+            toy_ec_txt,
+            'exts_list = [',
+            '   ("bar", "0.0", {',
+            '       "sources": "bar-%(version)s.tar.gz",',
+            '   }),',
+            ']',
+        ])
+        write_file(test_ec, test_ec_txt)
+        self.test_toy_build(ec_file=test_ec)
+
+    def test_toy_extension_sources_git_config(self):
+        """Test install toy that includes extensions with 'sources' spec including 'git_config'."""
+        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
+        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
+        toy_ec_txt = read_file(toy_ec)
+
+        # Tar-ball which should be created via 'git_config', and one file
+        ext_tgz = 'exts-git.tar.gz'
+        ext_tarball = os.path.join(self.test_sourcepath, 't', 'toy', ext_tgz)
+        ext_tarfile = 'a_directory/a_file.txt'
+
+        # Dummy source code required for extensions build_step to pass
+        ext_code = 'int main() { return 0; }'
+        ext_cfile = 'exts-git.c'
+
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        test_ec_txt = '\n'.join([
+            toy_ec_txt,
+            'prebuildopts = "echo \\\"%s\\\" > %s && ",' % (ext_code, ext_cfile),
+            'exts_list = [',
+            '   ("exts-git", "0.0", {',
+            '       "buildopts": "&& ls -l %s %s",' % (ext_tarball, ext_tarfile),
+            '       "sources": {',
+            '           "filename": "%(name)s.tar.gz",',
+            '           "git_config": {',
+            '               "repo_name": "testrepository",',
+            '               "url": "https://github.com/easybuilders",',
+            '               "tag": "master",',
+            '           },',
+            '       },',
+            '   }),',
+            ']',
+        ])
+        write_file(test_ec, test_ec_txt)
+        self.test_toy_build(ec_file=test_ec)
+
     def test_toy_module_fulltxt(self):
         """Strict text comparison of generated module file."""
         self.test_toy_tweaked()