diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 334462df0..955efd73b 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -125,6 +125,8 @@ def __init__( self.sandbox: Optional[MappingNode] = None self.splits: Optional[MappingNode] = None + self.source_provenance_fields: Optional[MappingNode] = None # Source provenance fields and their description + # # Private members # @@ -726,6 +728,7 @@ def _validate_toplevel_node(self, node, *, first_pass=False): "sources", "source-caches", "junctions", + "source-provenance-fields", "(@)", "(?)", ] @@ -1005,6 +1008,7 @@ def _load_second_pass(self): mount = _HostMount(path, host_path, optional) self._shell_host_files.append(mount) + self.source_provenance_fields = config.get_mapping("source-provenance-fields") # _load_pass(): # diff --git a/src/buildstream/data/projectconfig.yaml b/src/buildstream/data/projectconfig.yaml index 4632d7a1d..774e6056a 100644 --- a/src/buildstream/data/projectconfig.yaml +++ b/src/buildstream/data/projectconfig.yaml @@ -175,6 +175,12 @@ shell: # command: [ 'sh', '-i' ] +# Define the set of fields accepted in `provenance` dictionaries of sources. +# +source-provenance-fields: + homepage: "The project homepage URL" + issue-tracker: "The project's issue tracking URL" + # Defaults for bst commands # defaults: diff --git a/src/buildstream/element.py b/src/buildstream/element.py index fa79c2b44..007321139 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -85,12 +85,12 @@ from . import utils from . import _cachekey from . import _site -from .node import Node +from .node import Node, MappingNode from .plugin import Plugin from .sandbox import _SandboxFlags, SandboxCommandError from .sandbox._config import SandboxConfig from .sandbox._sandboxremote import SandboxRemote -from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance +from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey from ._artifact import Artifact from ._elementproxy import ElementProxy from ._elementsources import ElementSources @@ -102,7 +102,7 @@ if TYPE_CHECKING: from typing import Tuple - from .node import MappingNode, ScalarNode, SequenceNode + from .node import ScalarNode, SequenceNode from .types import SourceRef # pylint: disable=cyclic-import @@ -2635,11 +2635,19 @@ def __load_sources(self, load_element): del source[Symbol.DIRECTORY] # Provenance is optional - provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None) - provenance = None + provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None) if provenance_node: del source[Symbol.PROVENANCE] - provenance = _SourceProvenance.new_from_node(provenance_node) + defined_attrs = project._project_conf.get_mapping("source-provenance-fields", None) or project.source_provenance_fields + try: + provenance_node.validate_keys(defined_attrs.keys()) + except LoadError as E: + raise LoadError( + "Specified source attribute not defined in project config\n {}".format(E), + LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE) + + # make sure everything is a string + provenance_node = MappingNode.from_dict({key: value.as_str() for key, value in provenance_node.items()}) meta_source = MetaSource( self.name, @@ -2647,7 +2655,7 @@ def __load_sources(self, load_element): self.get_kind(), kind.as_str(), directory, - provenance, + provenance_node, source, load_element.first_pass, ) diff --git a/src/buildstream/exceptions.py b/src/buildstream/exceptions.py index 4806218e9..418e51eda 100644 --- a/src/buildstream/exceptions.py +++ b/src/buildstream/exceptions.py @@ -155,3 +155,8 @@ class LoadErrorReason(Enum): This warning will be produced when a filename for a target contains invalid characters in its name. """ + + UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29 + """ + Thee source provenance attribute specified was not defined in the project config + """ diff --git a/src/buildstream/source.py b/src/buildstream/source.py index 0ad4b612b..8cdf25208 100644 --- a/src/buildstream/source.py +++ b/src/buildstream/source.py @@ -38,15 +38,11 @@ to provide additional source provenance related metadata which will later be reported in :class:`.SourceInfo` objects. - The ``provenance`` dictionary supports the following fields: + The ``provenance`` dictionary itself does not have any specific required keys. - * Homepage - - The ``homepage`` attribute can be used to specify the project homepage URL - - * Issue Tracker - - The ``issue-tracker`` attribute can be used to specify the project's issue tracking URL + Any attribute used in the ``provenance`` dictionary of a source must be + defined in the project.conf using the ``source-provenance-fields`` dictionary + to define the attribute and its significance. *Since: 2.5* @@ -378,9 +374,9 @@ from .node import MappingNode from .plugin import Plugin from .sourcemirror import SourceMirror -from .types import SourceRef, CoreWarnings, FastEnum, _SourceProvenance -from ._exceptions import BstError, ImplError, PluginError -from .exceptions import ErrorDomain +from .types import SourceRef, CoreWarnings, FastEnum +from ._exceptions import BstError, ImplError, PluginError, LoadError +from .exceptions import ErrorDomain, LoadErrorReason from ._loader.metasource import MetaSource from ._projectrefs import ProjectRefStorage from ._cachekey import generate_key @@ -396,6 +392,8 @@ # pylint: enable=cyclic-import +SourceProvenance = MappingNode + class SourceError(BstError): """This exception should be raised by :class:`.Source` implementations @@ -553,8 +551,7 @@ def __init__( self, kind: str, url: str, - homepage: Optional[str], - issue_tracker: Optional[str], + provenance: Optional[SourceProvenance], medium: Union[SourceInfoMedium, str], version_type: Union[SourceVersionType, str], version: str, @@ -572,14 +569,9 @@ def __init__( The url of the source input """ - self.homepage: Optional[str] = homepage + self.provenance = provenance """ - The project homepage URL - """ - - self.issue_tracker: Optional[str] = issue_tracker - """ - The project issue tracking URL + The optional YAML node with source provenance attributes """ self.medium: Union[SourceInfoMedium, str] = medium @@ -642,10 +634,14 @@ def serialize(self) -> Dict[str, Union[str, Dict[str, str]]]: "url": self.url, } - if self.homepage is not None: - version_info["homepage"] = self.homepage - if self.issue_tracker is not None: - version_info["issue-tracker"] = self.issue_tracker + if self.provenance is not None: + # need to keep homepage/issue-tracker [also] at the top-level for backward compat + if (homepage := self.provenance.get_str("homepage", None)) is not None: + version_info["homepage"] = homepage + if (issue_tracker := self.provenance.get_str("issue-tracker", None)) is not None: + version_info["issue-tracker"] = issue_tracker + + version_info["provenance"] = self.provenance version_info["medium"] = medium_str version_info["version-type"] = version_type_str @@ -824,9 +820,9 @@ def __init__( self.__element_kind = meta.element_kind # The kind of the element owning this source self._directory = meta.directory # Staging relative directory self.__variables = variables # The variables used to resolve the source's config - self.__provenance: Optional[ - _SourceProvenance - ] = meta.provenance # The _SourceProvenance for general user provided SourceInfo + self.__provenance: Optional[SourceProvenance] = ( + meta.provenance + ) # The source provenance for general user provided SourceInfo self.__key = None # Cache key for source @@ -1367,6 +1363,7 @@ def create_source_info( version: str, *, version_guess: Optional[str] = None, + provenance_node: Optional[MappingNode] = None, extra_data: Optional[Dict[str, str]] = None, ) -> SourceInfo: """Create a :class:`.SourceInfo` object @@ -1386,21 +1383,39 @@ def create_source_info( choice depicting the type of version. version: A string which represents a unique version of this source input version_guess: An optional string representing the guessed human readable version + provenance_node: The optional YAML node with source provenance attributes, + defaults to the provenance specified at the top level of the source. extra_data: Additional plugin defined key/values *Since: 2.5* """ - homepage = None - issue_tracker = None - if self.__provenance is not None: - homepage = self.__provenance.homepage - issue_tracker = self.__provenance.issue_tracker + project = self._get_project() + + if provenance_node is not None: + # Ensure provenance node keys are valid and values are all strings + defined_provenance_fields = ( + project._project_conf.get_mapping("source-provenance-fields", None) + or project.source_provenance_fields + ) + + try: + provenance_node.validate_keys(defined_provenance_fields.keys()) + except LoadError as E: + raise LoadError( + "Specified source attribute not defined in project config\n {}".format(E), + LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE, + ) + + # Make sure everything is a string + provenance = MappingNode.from_dict({key: value.as_str() for key, value in provenance_node.items()}) + + else: + provenance = self.__provenance return SourceInfo( self.get_kind(), url, - homepage, - issue_tracker, + provenance, medium, version_type, version, diff --git a/src/buildstream/types.py b/src/buildstream/types.py index d4d5e3cdb..0cc2106b3 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -390,42 +390,6 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror": return cls(name, aliases) -# _SourceProvenance() -# -# A simple object describing user provided source provenance information -# -# Args: -# homepage: The project homepage URL -# issue_tracker: The project issue reporting URL -# -class _SourceProvenance: - def __init__(self, homepage: Optional[str], issue_tracker: Optional[str]): - self.homepage: Optional[str] = homepage - self.issue_tracker: Optional[str] = issue_tracker - - # new_from_node(): - # - # Creates a _SourceProvenance() from a YAML loaded node. - # - # Args: - # node: The configuration node describing the spec. - # - # Returns: - # The described _SourceProvenance instance. - # - # Raises: - # LoadError: If the node is malformed. - # - @classmethod - def new_from_node(cls, node: MappingNode) -> "_SourceProvenance": - node.validate_keys(["homepage", "issue-tracker"]) - - homepage: Optional[str] = node.get_str("homepage", None) - issue_tracker: Optional[str] = node.get_str("issue-tracker", None) - - return cls(homepage, issue_tracker) - - ######################################## # Type aliases # ######################################## diff --git a/tests/frontend/show.py b/tests/frontend/show.py index 6ad762224..5b91d7120 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -814,3 +814,33 @@ def test_source_info_workspace(cli, datafiles, tmpdir): # There is no version guessing for a workspace assert source_info.get_str("version-guess", None) is None + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, "source-info")) +def test_multi_source_info(cli, datafiles): + project = str(datafiles) + result = cli.run( + project=project, silent=True, args=["show", "--format", "%{name}:\n%{source-info}", "multisource.bst"] + ) + result.assert_success() + + loaded = _yaml.load_data(result.output) + sources = loaded.get_sequence("multisource.bst") + + source_info = sources.mapping_at(0) + assert source_info.get_str("kind") == "multisource" + assert source_info.get_str("url") == "http://ponyfarm.com/ponies" + assert source_info.get_str("medium") == "pony-ride" + assert source_info.get_str("version-type") == "pony-age" + assert source_info.get_str("version") == "1234567" + assert source_info.get_str("version-guess", None) == "12" + assert source_info.get_str("homepage") == "https://flying-ponies.com/index.html" + + source_info = sources.mapping_at(1) + assert source_info.get_str("kind") == "multisource" + assert source_info.get_str("url") == "http://ponyfarm.com/happy" + assert source_info.get_str("medium") == "pony-ride" + assert source_info.get_str("version-type") == "pony-age" + assert source_info.get_str("version") == "1234567" + assert source_info.get_str("version-guess", None) == "12" + assert source_info.get_str("homepage") == "http://happy.ponyfarm.com" diff --git a/tests/frontend/source-info/elements/multisource.bst b/tests/frontend/source-info/elements/multisource.bst new file mode 100644 index 000000000..7487548ed --- /dev/null +++ b/tests/frontend/source-info/elements/multisource.bst @@ -0,0 +1,6 @@ +kind: import + +sources: +- kind: multisource + provenance: + homepage: https://flying-ponies.com/index.html diff --git a/tests/frontend/source-info/plugins/multisource.py b/tests/frontend/source-info/plugins/multisource.py new file mode 100644 index 000000000..0f1b918d9 --- /dev/null +++ b/tests/frontend/source-info/plugins/multisource.py @@ -0,0 +1,46 @@ +from buildstream import Node, Source + + +class MultiSource(Source): + BST_MIN_VERSION = "2.0" + + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + return {} + + def load_ref(self, node): + pass + + def get_ref(self): + return {} + + def set_ref(self, ref, node): + pass + + def is_cached(self): + return False + + def collect_source_info(self): + return [ + self.create_source_info( + "http://ponyfarm.com/ponies", "pony-ride", "pony-age", "1234567", version_guess="12" + ), + self.create_source_info( + "http://ponyfarm.com/happy", + "pony-ride", + "pony-age", + "1234567", + version_guess="12", + provenance_node=Node.from_dict({"homepage": "http://happy.ponyfarm.com"}), + ), + ] + + +# Plugin entry point +def setup(): + return MultiSource diff --git a/tests/frontend/source-info/project.conf b/tests/frontend/source-info/project.conf index 86cfd087b..3ba58ad6f 100644 --- a/tests/frontend/source-info/project.conf +++ b/tests/frontend/source-info/project.conf @@ -12,5 +12,6 @@ plugins: path: plugins sources: - extradata + - multisource - testsource - unimplemented