Merge pull request #2098 from joshua-zivkovic/JZ/multi-source-provenance

Support multi source source provenance info

Author: juergbi

src/buildstream/source.py

@@ -759,6 +759,14 @@ class Source(Plugin):
     # The defaults from the project
     __defaults: Optional[Dict[str, Any]] = None
 
+    BST_CUSTOM_SOURCE_PROVENANCE = False
+    """Whether multiple sources' provenance information are provided
+
+    Used primarily to override the usage of top-level source
+    provenance of a source where individual sub-source's
+    provenance should instead be provided
+    """
+
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = False
     """Whether access to previous sources is required during track
 
@@ -828,6 +836,12 @@ def __init__(
             _SourceProvenance
         ] = meta.provenance  # The _SourceProvenance for general user provided SourceInfo
 
+        if self.__provenance is not None and self.BST_CUSTOM_SOURCE_PROVENANCE:
+            raise SourceError(
+                f"{self._get_provenance()} Custom source provenance plugin: Refusing to use top level source provenance",
+                reason="top-level-provenance-on-custom-implementation",
+            )
+
         self.__key = None  # Cache key for source
 
         # The alias_override is only set on a re-instantiated Source
@@ -1368,6 +1382,7 @@ def create_source_info(
         *,
         version_guess: Optional[str] = None,
         extra_data: Optional[Dict[str, str]] = None,
+        provenance_node: Optional[MappingNode] = None,
     ) -> SourceInfo:
         """Create a :class:`.SourceInfo` object
 
@@ -1387,14 +1402,22 @@ def create_source_info(
            version: A string which represents a unique version of this source input
            version_guess: An optional string representing the guessed human readable version
            extra_data: Additional plugin defined key/values
+           provenance_node: An optional :class:`Node <buildstream.node.Node>` with source provenance attributes,
+                            defaults to the provenance specified at the top level of the source.
 
         *Since: 2.5*
         """
         homepage = None
         issue_tracker = None
-        if self.__provenance is not None:
-            homepage = self.__provenance.homepage
-            issue_tracker = self.__provenance.issue_tracker
+
+        if provenance_node is not None:
+            source_provenance: Optional[_SourceProvenance] = _SourceProvenance.new_from_node(provenance_node)
+        else:
+            source_provenance = self.__provenance
+
+        if source_provenance is not None:
+            homepage = source_provenance.homepage
+            issue_tracker = source_provenance.issue_tracker
 
         return SourceInfo(
             self.get_kind(),

tests/frontend/show.py

@@ -814,3 +814,33 @@ def test_source_info_workspace(cli, datafiles, tmpdir):
 
     # There is no version guessing for a workspace
     assert source_info.get_str("version-guess", None) is None
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info"))
+def test_multi_source_info(cli, datafiles):
+    project = str(datafiles)
+    result = cli.run(
+        project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "multisource.bst"]
+    )
+    result.assert_success()
+
+    loaded = _yaml.load_data(result.output)
+    sources = loaded.get_sequence("multisource.bst")
+
+    source_info = sources.mapping_at(0)
+    assert source_info.get_str("kind") == "multisource"
+    assert source_info.get_str("url") == "http://ponyfarm.com/ponies"
+    assert source_info.get_str("medium") == "pony-ride"
+    assert source_info.get_str("version-type") == "pony-age"
+    assert source_info.get_str("version") == "1234567"
+    assert source_info.get_str("version-guess", None) == "12"
+    assert "homepage" not in source_info
+
+    source_info = sources.mapping_at(1)
+    assert source_info.get_str("kind") == "multisource"
+    assert source_info.get_str("url") == "http://ponyfarm.com/happy"
+    assert source_info.get_str("medium") == "pony-ride"
+    assert source_info.get_str("version-type") == "pony-age"
+    assert source_info.get_str("version") == "1234567"
+    assert source_info.get_str("version-guess", None) == "12"
+    assert source_info.get_str("homepage") == "http://happy.ponyfarm.com"

tests/frontend/source-info/elements/multisource.bst

@@ -0,0 +1,4 @@
+kind: import
+
+sources:
+- kind: multisource

tests/frontend/source-info/plugins/multisource.py

@@ -0,0 +1,48 @@
+from buildstream import Node, Source
+
+
+class MultiSource(Source):
+    BST_MIN_VERSION = "2.0"
+
+    BST_CUSTOM_SOURCE_PROVENANCE = True
+
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        return {}
+
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return {}
+
+    def set_ref(self, ref, node):
+        pass
+
+    def is_cached(self):
+        return False
+
+    def collect_source_info(self):
+        return [
+            self.create_source_info(
+                "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12"
+            ),
+            self.create_source_info(
+                "http://ponyfarm.com/happy",
+                "pony-ride",
+                "pony-age",
+                "1234567",
+                version_guess="12",
+                provenance_node=Node.from_dict({"homepage": "http://happy.ponyfarm.com"}),
+            ),
+        ]
+
+
+# Plugin entry point
+def setup():
+    return MultiSource

tests/frontend/source-info/project.conf

@@ -12,5 +12,6 @@ plugins:
   path: plugins
   sources:
   - extradata
+  - multisource
   - testsource
   - unimplemented

tests/sources/source_provenance_attributes.py

@@ -0,0 +1,51 @@
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+import pytest
+
+from buildstream._testing import generate_project, load_yaml
+from buildstream._testing import cli  # pylint: disable=unused-import
+from buildstream.exceptions import ErrorDomain
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "source_provenance_attributes")
+
+
+##################################################################
+#                              Tests                             #
+##################################################################
+# Test that no defined source provenance attributes blocks all source provenance data
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_provenance_disallow_top_level(cli, datafiles):
+    project = str(datafiles)
+
+    # Set the project_dir alias in project.conf to the path to the tested project
+    project_config_path = os.path.join(project, "project.conf")
+    project_config = load_yaml(project_config_path)
+    aliases = project_config.get_mapping("aliases")
+    aliases["project_dir"] = "file://{}".format(project)
+
+    generate_project(project, project_config)
+
+    # Make sure disallowed usage of top-level source proveance fails
+    result = cli.run(
+        project=project,
+        args=["show", "target.bst"],
+    )
+
+    result.assert_main_error(ErrorDomain.SOURCE, "top-level-provenance-on-custom-implementation")

tests/sources/source_provenance_attributes/elements/target.bst

@@ -0,0 +1,7 @@
+kind: import
+
+sources:
+- kind: multisource-plugin
+  url: project_dir:/files/file
+  provenance:
+    homepage: bar

tests/sources/source_provenance_attributes/files/file

@@ -0,0 +1 @@
+Hello World!

tests/sources/source_provenance_attributes/plugins/multisource-plugin.py

@@ -0,0 +1,36 @@
+from buildstream import Node, Source
+
+
+class MultiSource(Source):
+    BST_MIN_VERSION = "2.0"
+
+    BST_CUSTOM_SOURCE_PROVENANCE = True
+
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        return {}
+
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return {}
+
+    def set_ref(self, ref, node):
+        pass
+
+    def is_cached(self):
+        return False
+
+    def collect_source_info(self):
+        return []
+
+
+# Plugin entry point
+def setup():
+    return MultiSource

tests/sources/source_provenance_attributes/project.conf

@@ -0,0 +1,14 @@
+# Project with source provenance attributes
+name: foo
+min-version: 2.0
+
+element-path: elements
+
+aliases:
+  project_dir: file://{project_dir}
+
+plugins:
+- origin: local
+  path: plugins
+  sources:
+  - multisource-plugin