Generic Source Provenance

src/buildstream/_project.py

@@ -125,6 +125,8 @@ def __init__(
         self.sandbox: Optional[MappingNode] = None
         self.splits: Optional[MappingNode] = None
 
+        self.source_provenance_fields: Optional[MappingNode] = None  # Source provenance fields and their description
+
         #
         # Private members
         #
@@ -726,6 +728,7 @@ def _validate_toplevel_node(self, node, *, first_pass=False):
                 "sources",
                 "source-caches",
                 "junctions",
+                "source-provenance-fields",
                 "(@)",
                 "(?)",
             ]
@@ -1005,6 +1008,7 @@ def _load_second_pass(self):
                 mount = _HostMount(path, host_path, optional)
 
             self._shell_host_files.append(mount)
+        self.source_provenance_fields = config.get_mapping("source-provenance-fields")
 
     # _load_pass():
     #

src/buildstream/data/projectconfig.yaml

@@ -175,6 +175,12 @@ shell:
   #
   command: [ 'sh', '-i' ]
 
+# Define the set of fields accepted in `provenance` dictionaries of sources.
+#
+source-provenance-fields:
+  homepage: "The project homepage URL"
+  issue-tracker: "The project's issue tracking URL"
+
 # Defaults for bst commands
 #
 defaults:

src/buildstream/element.py

@@ -85,12 +85,12 @@
 from . import utils
 from . import _cachekey
 from . import _site
-from .node import Node
+from .node import Node, MappingNode
 from .plugin import Plugin
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -102,7 +102,7 @@
 
 if TYPE_CHECKING:
     from typing import Tuple
-    from .node import MappingNode, ScalarNode, SequenceNode
+    from .node import ScalarNode, SequenceNode
     from .types import SourceRef
 
     # pylint: disable=cyclic-import
@@ -2635,19 +2635,27 @@ def __load_sources(self, load_element):
                     del source[Symbol.DIRECTORY]
 
                 # Provenance is optional
-                provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None)
-                provenance = None
+                provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None)
                 if provenance_node:
                     del source[Symbol.PROVENANCE]
-                    provenance = _SourceProvenance.new_from_node(provenance_node)
+                    defined_attrs = project._project_conf.get_mapping("source-provenance-fields", None) or project.source_provenance_fields
+                    try:
+                        provenance_node.validate_keys(defined_attrs.keys())
+                    except LoadError as E:
+                        raise LoadError(
+                            "Specified source attribute not defined in project config\n {}".format(E),
+                            LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE)
+
+                    # make sure everything is a string
+                    provenance_node = MappingNode.from_dict({key: value.as_str() for key, value in provenance_node.items()})
 
                 meta_source = MetaSource(
                     self.name,
                     index,
                     self.get_kind(),
                     kind.as_str(),
                     directory,
-                    provenance,
+                    provenance_node,
                     source,
                     load_element.first_pass,
                 )

src/buildstream/exceptions.py

@@ -155,3 +155,8 @@ class LoadErrorReason(Enum):
     This warning will be produced when a filename for a target contains invalid
     characters in its name.
     """
+
+    UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29
+    """
+    Thee source provenance attribute specified was not defined in the project config
+    """

src/buildstream/source.py

@@ -38,15 +38,11 @@
   to provide additional source provenance related metadata which will later
   be reported in :class:`.SourceInfo` objects.
 
-  The ``provenance`` dictionary supports the following fields:
+  The ``provenance`` dictionary itself does not have any specific required keys.
 
-  * Homepage
-
-    The ``homepage`` attribute can be used to specify the project homepage URL
-
-  * Issue Tracker
-
-    The ``issue-tracker`` attribute can be used to specify the project's issue tracking URL
+  Any attribute used in the ``provenance`` dictionary of a source must be
+  defined in the project.conf using the ``source-provenance-fields`` dictionary
+  to define the attribute and its significance.
 
   *Since: 2.5*
 
@@ -378,9 +374,9 @@
 from .node import MappingNode
 from .plugin import Plugin
 from .sourcemirror import SourceMirror
-from .types import SourceRef, CoreWarnings, FastEnum, _SourceProvenance
-from ._exceptions import BstError, ImplError, PluginError
-from .exceptions import ErrorDomain
+from .types import SourceRef, CoreWarnings, FastEnum
+from ._exceptions import BstError, ImplError, PluginError, LoadError
+from .exceptions import ErrorDomain, LoadErrorReason
 from ._loader.metasource import MetaSource
 from ._projectrefs import ProjectRefStorage
 from ._cachekey import generate_key
@@ -396,6 +392,8 @@
 
     # pylint: enable=cyclic-import
 
+SourceProvenance = MappingNode
+
 
 class SourceError(BstError):
     """This exception should be raised by :class:`.Source` implementations
@@ -553,8 +551,7 @@ def __init__(
         self,
         kind: str,
         url: str,
-        homepage: Optional[str],
-        issue_tracker: Optional[str],
+        provenance: Optional[SourceProvenance],
         medium: Union[SourceInfoMedium, str],
         version_type: Union[SourceVersionType, str],
         version: str,
@@ -572,14 +569,9 @@ def __init__(
         The url of the source input
         """
 
-        self.homepage: Optional[str] = homepage
+        self.provenance = provenance
         """
-        The project homepage URL
-        """
-
-        self.issue_tracker: Optional[str] = issue_tracker
-        """
-        The project issue tracking URL
+        The optional YAML node with source provenance attributes
         """
 
         self.medium: Union[SourceInfoMedium, str] = medium
@@ -642,10 +634,14 @@ def serialize(self) -> Dict[str, Union[str, Dict[str, str]]]:
             "url": self.url,
         }
 
-        if self.homepage is not None:
-            version_info["homepage"] = self.homepage
-        if self.issue_tracker is not None:
-            version_info["issue-tracker"] = self.issue_tracker
+        if self.provenance is not None:
+            # need to keep homepage/issue-tracker [also] at the top-level for backward compat
+            if (homepage := self.provenance.get_str("homepage", None)) is not None:
+                version_info["homepage"] = homepage
+            if (issue_tracker := self.provenance.get_str("issue-tracker", None)) is not None:
+                version_info["issue-tracker"] = issue_tracker
+
+            version_info["provenance"] = self.provenance
 
         version_info["medium"] = medium_str
         version_info["version-type"] = version_type_str
@@ -824,9 +820,9 @@ def __init__(
         self.__element_kind = meta.element_kind  # The kind of the element owning this source
         self._directory = meta.directory  # Staging relative directory
         self.__variables = variables  # The variables used to resolve the source's config
-        self.__provenance: Optional[
-            _SourceProvenance
-        ] = meta.provenance  # The _SourceProvenance for general user provided SourceInfo
+        self.__provenance: Optional[SourceProvenance] = (
+            meta.provenance
+        )  # The source provenance for general user provided SourceInfo
 
         self.__key = None  # Cache key for source
 
@@ -1367,6 +1363,7 @@ def create_source_info(
         version: str,
         *,
         version_guess: Optional[str] = None,
+        provenance_node: Optional[MappingNode] = None,
         extra_data: Optional[Dict[str, str]] = None,
     ) -> SourceInfo:
         """Create a :class:`.SourceInfo` object
@@ -1386,21 +1383,39 @@ def create_source_info(
                          choice depicting the type of version.
            version: A string which represents a unique version of this source input
            version_guess: An optional string representing the guessed human readable version
+           provenance_node: The optional YAML node with source provenance attributes,
+                            defaults to the provenance specified at the top level of the source.
            extra_data: Additional plugin defined key/values
 
         *Since: 2.5*
         """
-        homepage = None
-        issue_tracker = None
-        if self.__provenance is not None:
-            homepage = self.__provenance.homepage
-            issue_tracker = self.__provenance.issue_tracker
+        project = self._get_project()
+
+        if provenance_node is not None:
+            # Ensure provenance node keys are valid and values are all strings
+            defined_provenance_fields = (
+                project._project_conf.get_mapping("source-provenance-fields", None)
+                or project.source_provenance_fields
+            )
+
+            try:
+                provenance_node.validate_keys(defined_provenance_fields.keys())
+            except LoadError as E:
+                raise LoadError(
+                    "Specified source attribute not defined in project config\n {}".format(E),
+                    LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE,
+                )
+
+            # Make sure everything is a string
+            provenance = MappingNode.from_dict({key: value.as_str() for key, value in provenance_node.items()})
+
+        else:
+            provenance = self.__provenance
 
         return SourceInfo(
             self.get_kind(),
             url,
-            homepage,
-            issue_tracker,
+            provenance,
             medium,
             version_type,
             version,

src/buildstream/types.py

@@ -390,42 +390,6 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror":
         return cls(name, aliases)
 
 
-# _SourceProvenance()
-#
-# A simple object describing user provided source provenance information
-#
-# Args:
-#    homepage: The project homepage URL
-#    issue_tracker: The project issue reporting URL
-#
-class _SourceProvenance:
-    def __init__(self, homepage: Optional[str], issue_tracker: Optional[str]):
-        self.homepage: Optional[str] = homepage
-        self.issue_tracker: Optional[str] = issue_tracker
-
-    # new_from_node():
-    #
-    # Creates a _SourceProvenance() from a YAML loaded node.
-    #
-    # Args:
-    #    node: The configuration node describing the spec.
-    #
-    # Returns:
-    #    The described _SourceProvenance instance.
-    #
-    # Raises:
-    #    LoadError: If the node is malformed.
-    #
-    @classmethod
-    def new_from_node(cls, node: MappingNode) -> "_SourceProvenance":
-        node.validate_keys(["homepage", "issue-tracker"])
-
-        homepage: Optional[str] = node.get_str("homepage", None)
-        issue_tracker: Optional[str] = node.get_str("issue-tracker", None)
-
-        return cls(homepage, issue_tracker)
-
-
 ########################################
 #           Type aliases               #
 ########################################

tests/frontend/show.py

@@ -814,3 +814,33 @@ def test_source_info_workspace(cli, datafiles, tmpdir):
 
     # There is no version guessing for a workspace
     assert source_info.get_str("version-guess", None) is None
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info"))
+def test_multi_source_info(cli, datafiles):
+    project = str(datafiles)
+    result = cli.run(
+        project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "multisource.bst"]
+    )
+    result.assert_success()
+
+    loaded = _yaml.load_data(result.output)
+    sources = loaded.get_sequence("multisource.bst")
+
+    source_info = sources.mapping_at(0)
+    assert source_info.get_str("kind") == "multisource"
+    assert source_info.get_str("url") == "http://ponyfarm.com/ponies"
+    assert source_info.get_str("medium") == "pony-ride"
+    assert source_info.get_str("version-type") == "pony-age"
+    assert source_info.get_str("version") == "1234567"
+    assert source_info.get_str("version-guess", None) == "12"
+    assert source_info.get_str("homepage") == "https://flying-ponies.com/index.html"
+
+    source_info = sources.mapping_at(1)
+    assert source_info.get_str("kind") == "multisource"
+    assert source_info.get_str("url") == "http://ponyfarm.com/happy"
+    assert source_info.get_str("medium") == "pony-ride"
+    assert source_info.get_str("version-type") == "pony-age"
+    assert source_info.get_str("version") == "1234567"
+    assert source_info.get_str("version-guess", None) == "12"
+    assert source_info.get_str("homepage") == "http://happy.ponyfarm.com"

tests/frontend/source-info/elements/multisource.bst

@@ -0,0 +1,6 @@
+kind: import
+
+sources:
+- kind: multisource
+  provenance:
+    homepage: https://flying-ponies.com/index.html

tests/frontend/source-info/plugins/multisource.py

@@ -0,0 +1,46 @@
+from buildstream import Node, Source
+
+
+class MultiSource(Source):
+    BST_MIN_VERSION = "2.0"
+
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        return {}
+
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return {}
+
+    def set_ref(self, ref, node):
+        pass
+
+    def is_cached(self):
+        return False
+
+    def collect_source_info(self):
+        return [
+            self.create_source_info(
+                "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12"
+            ),
+            self.create_source_info(
+                "http://ponyfarm.com/happy",
+                "pony-ride",
+                "pony-age",
+                "1234567",
+                version_guess="12",
+                provenance_node=Node.from_dict({"homepage": "http://happy.ponyfarm.com"}),
+            ),
+        ]
+
+
+# Plugin entry point
+def setup():
+    return MultiSource

tests/frontend/source-info/project.conf

@@ -12,5 +12,6 @@ plugins:
   path: plugins
   sources:
   - extradata
+  - multisource
   - testsource
   - unimplemented