manifest: support URL modification in downstream manifests

Add support for a new 'import-modifications' section in the manifest.

Under the 'url-replace' key, users can define search-and-replace
patterns that are applied to project URLs during manifest import.
This allows downstream projects to modify remote URLs, e.g. when using
mirrored repositories.

Author: thorsten-klein

src/west/manifest-schema.yml

@@ -66,6 +66,25 @@ mapping:
             required: true
             type: str
 
+  # allow some modification of imported projects by downstream projects
+  import-modifications:
+    required: false
+    type: map
+    mapping:
+      # search-replace within URLs (e.g. to use mirror URLs)
+      url-replace:
+        required: false
+        type: seq
+        sequence:
+          - type: map
+            mapping:
+              old:
+                required: true
+                type: str
+              new:
+                required: true
+                type: str
+
   # The "projects" key specifies a sequence of "projects", each of which has a
   # remote, and may specify additional configuration.
   #

src/west/manifest.py

@@ -7,6 +7,7 @@
 Parser and abstract data types for west manifests.
 '''
 
+import copy
 import enum
 import errno
 import logging
@@ -182,6 +183,34 @@ def _err(message):
 _logger = logging.getLogger(__name__)
 
 
+# Representation of import-modifications
+
+
+class ImportModifications:
+    """Represents `import-modifications` within a manifest."""
+
+    def __init__(self, manifest_data: dict | None = None):
+        """Initialize a new ImportModifications instance."""
+        self.url_replaces: list[tuple[str, str]] = []
+        self.append(manifest_data or {})
+
+    def append(self, manifest_data: dict):
+        """Append values from a manifest data (dictionary) to this instance."""
+        for kind, values in manifest_data.get('import-modifications', {}).items():
+            if kind == 'url-replace':
+                self.url_replaces += [(v['old'], v['new']) for v in values]
+
+    def merge(self, other):
+        """Merge another ImportModifications instance into this one."""
+        if not isinstance(other, ImportModifications):
+            raise TypeError(f"Unsupported type'{type(other).__name__}'")
+        self.url_replaces += other.url_replaces
+
+    def copy(self):
+        """Return a deep copy of this instance."""
+        return copy.deepcopy(self)
+
+
 # Type for the submodule value passed through the manifest file.
 class Submodule(NamedTuple):
     '''Represents a Git submodule within a project.'''
@@ -456,6 +485,9 @@ class _import_ctx(NamedTuple):
     # Bit vector of flags that modify import behavior.
     import_flags: 'ImportFlag'
 
+    # import-modifications
+    modifications: ImportModifications
+
 
 def _imap_filter_allows(imap_filter: ImapFilterFnType, project: 'Project') -> bool:
     # imap_filter(project) if imap_filter is not None; True otherwise.
@@ -2052,6 +2084,7 @@ def get_option(option, default=None):
             current_repo_abspath=current_repo_abspath,
             project_importer=project_importer,
             import_flags=import_flags,
+            modifications=ImportModifications(),
         )
 
     def _recursive_init(self, ctx: _import_ctx):
@@ -2074,6 +2107,10 @@ def _load_validated(self) -> None:
 
         manifest_data = self._ctx.current_data['manifest']
 
+        # append values from resolved manifest_data to current context
+        new_modifications = ImportModifications(manifest_data)
+        self._ctx.modifications.merge(new_modifications)
+
         schema_version = str(manifest_data.get('version', SCHEMA_VERSION))
 
         # We want to make an ordered map from project names to
@@ -2322,6 +2359,7 @@ def _import_pathobj_from_self(self, pathobj_abs: Path, pathobj: Path) -> None:
             current_abspath=pathobj_abs,
             current_relpath=pathobj,
             current_data=pathobj_abs.read_text(encoding=Manifest.encoding),
+            modifications=self._ctx.modifications.copy(),
         )
         try:
             Manifest(topdir=self.topdir, internal_import_ctx=child_ctx)
@@ -2452,6 +2490,13 @@ def _load_project(self, pd: dict, url_bases: dict[str, str], defaults: _defaults
         else:
             self._malformed(f'project {name} has no remote or url and no default remote is set')
 
+        # modify the url
+        if url:
+            url_replaces = self._ctx.modifications.url_replaces
+            for url_replace in reversed(url_replaces):
+                old, new = url_replace
+                url = url.replace(old, new)
+
         # The project's path needs to respect any import: path-prefix,
         # regardless of self._ctx.import_flags. The 'ignore' type flags
         # just mean ignore the imported data. The path-prefix in this
@@ -2672,6 +2717,7 @@ def _import_data_from_project(
             # We therefore use a separate list for tracking them
             # from our current list.
             manifest_west_commands=[],
+            modifications=self._ctx.modifications.copy(),
         )
         try:
             submanifest = Manifest(topdir=self.topdir, internal_import_ctx=child_ctx)