Implement tracking for source provenance

Expand source tracking to also be able to work with
source provenance data. This provides source plugins
with the option of implementing source provenance
data gathering whilst tracking source refs.

Author: joshua-zivkovic

src/buildstream/downloadablefilesource.py

@@ -269,6 +269,7 @@ class DownloadableFileSource(Source):
     def configure(self, node):
         self.original_url = node.get_str("url")
         self.ref = node.get_str("ref", None)
+        self.source_provenance = node.get_mapping("provenance", None)
 
         extra_data = {}
         self.url = self.translate_url(self.original_url, extra_data=extra_data)
@@ -310,6 +311,15 @@ def get_ref(self):
     def set_ref(self, ref, node):
         node["ref"] = self.ref = ref
 
+    def load_source_provenance(self, node):
+        self.source_provenance = node.get_str("provenance", None)
+
+    def get_source_provenance(self):
+        return self.source_provenance
+
+    def set_source_provenance(self, source_provenance, node):
+        node["provenance"] = self.source_provenance = source_provenance
+
     def track(self):  # pylint: disable=arguments-differ
         # there is no 'track' field in the source to determine what/whether
         # or not to update refs, because tracking a ref is always a conscious
@@ -325,7 +335,7 @@ def track(self):  # pylint: disable=arguments-differ
             )
             self.warn("Potential man-in-the-middle attack!", detail=detail)
 
-        return new_ref
+        return new_ref, None
 
     def fetch(self):  # pylint: disable=arguments-differ
 

src/buildstream/exceptions.py

@@ -155,3 +155,8 @@ class LoadErrorReason(Enum):
     This warning will be produced when a filename for a target contains invalid
     characters in its name.
     """
+
+    UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29
+    """
+    Thee source provenance attribute specified was not defined in the project config
+    """

src/buildstream/plugins/sources/local.py

@@ -93,6 +93,15 @@ def get_ref(self):
     def set_ref(self, ref, node):
         pass  # pragma: nocover
 
+    def load_source_provenance(self, node):
+        pass
+
+    def get_source_provenance(self):
+        return None
+
+    def set_source_provenance(self, source_provenance, node):
+        pass
+
     def fetch(self):  # pylint: disable=arguments-differ
         # Nothing to do here for a local source
         pass  # pragma: nocover

src/buildstream/source.py

@@ -381,8 +381,8 @@
 from .plugin import Plugin
 from .sourcemirror import SourceMirror
 from .types import SourceRef, CoreWarnings, FastEnum
-from ._exceptions import BstError, ImplError, PluginError
-from .exceptions import ErrorDomain
+from ._exceptions import BstError, ImplError, PluginError, LoadError
+from .exceptions import ErrorDomain, LoadErrorReason
 from ._loader.metasource import MetaSource
 from ._projectrefs import ProjectRefStorage
 from ._cachekey import generate_key
@@ -929,7 +929,16 @@ def set_ref(self, ref, node):
         """
         raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind()))
 
-    def track(self, *, previous_sources_dir: Optional[str] = None) -> SourceRef:
+    def load_source_provenance(self, node: MappingNode) -> None:
+        pass
+
+    def get_source_provenance(self) -> SourceProvenance:
+        return None
+
+    def set_source_provenance(self, source_provenance: SourceProvenance):
+        pass
+
+    def track(self, *, previous_sources_dir: Optional[str] = None) -> SourceRef | tuple[SourceRef, SourceProvenance]:
         """Resolve a new ref from the plugin's track option
 
         Args:
@@ -954,7 +963,7 @@ def track(self, *, previous_sources_dir: Optional[str] = None) -> SourceRef:
         Working with the :ref:`source ref is discussed here <core_source_ref>`.
         """
         # Allow a non implementation
-        return None
+        return None, None
 
     def fetch(self, *, previous_sources_dir: Optional[str] = None) -> None:
         """Fetch remote sources and mirror them locally, ensuring at least
@@ -1787,13 +1796,40 @@ def process_value(action, container, path, key, new_value):
     #   previous_sources_dir (str): directory where previous sources are staged
     #
     def _track(self, previous_sources_dir: Optional[str] = None) -> SourceRef:
+        def verify_provenance_attributes(provenance: SourceProvenance):
+            if type(provenance) is list:
+                # produce a list of unique attrs
+                unique_entries = {attr: "" for single_provenance in provenance for attr in single_provenance.keys()}
+                used_attrs = unique_entries.keys()
+            else:
+                used_attrs = provenance.keys()
+
+            project = self._get_project()
+            defined_provenance_fields = (
+                project._project_conf.get_mapping("source-provenance-fields", None) or project.source_provenance_fields
+            )
+            undefined_attributes = list(set(used_attrs) - set(defined_provenance_fields.keys()))
+
+            if len(undefined_attributes) > 0:
+                self.warn(f"Required source attributes not defined in project config: {undefined_attributes}")
+
         if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK:
-            new_ref = self.__do_track(previous_sources_dir=previous_sources_dir)
+            r = self.__do_track(previous_sources_dir=previous_sources_dir)
+        else:
+            r = self.__do_track()
+
+        if type(r) is tuple:
+            new_ref, source_provenance = r
         else:
-            new_ref = self.__do_track()
+            new_ref = r
+            source_provenance = None
 
         current_ref = self.get_ref()  # pylint: disable=assignment-from-no-return
 
+        # in the case of multi-source plugins, set_ref might update the provenance with the new content,
+        # designed to use ref, so grab the old content first
+        current_provenance: list[MappingNode] = self.get_source_provenance()
+
         if new_ref is None:
             # No tracking, keep current ref
             new_ref = current_ref
@@ -1806,6 +1842,36 @@ def _track(self, previous_sources_dir: Optional[str] = None) -> SourceRef:
 
         self._generate_key()
 
+        if source_provenance is not None:
+            verify_provenance_attributes(source_provenance)
+
+        if current_provenance is not None:
+            if type(current_provenance) is list:
+                current_provenance = list(map(MappingNode.strip_node_info, current_provenance))
+            else:
+                current_provenance = current_provenance.strip_node_info()
+
+        if current_provenance != source_provenance:
+            self.info(f"Updated source provenance content for {self.name}")
+
+            context = self._get_context()
+            project = self._get_project()
+            toplevel = context.get_toplevel_project()
+            toplevel_refs = self._project_refs(toplevel)
+            provenance = self._get_provenance()
+
+            element_name = self.__element_name
+            element_idx = self.__element_index
+
+            node = {}
+            if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
+                node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True)
+
+            if project is toplevel and not node:
+                node = provenance._node
+
+            self.set_source_provenance(source_provenance, node)
+
         return new_ref
 
     # _requires_previous_sources()
@@ -2034,14 +2100,14 @@ def __do_track(self, **kwargs):
         for mirror in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)):
             new_source = self.__clone_for_uri(mirror)
             try:
-                ref = new_source.track(**kwargs)  # pylint: disable=assignment-from-none
+                r = new_source.track(**kwargs)  # pylint: disable=assignment-from-none
             # FIXME: Need to consider temporary vs. permanent failures,
             #        and how this works with retries.
             except BstError as e:
                 last_error = e
                 continue
 
-            return ref
+            return r
 
         raise last_error