cargo: create dataclasses for Cargo.lock

Start introducing a new simpler API for conversion of TypedDicts to
dataclasses, and use it already for Cargo.lock.

Signed-off-by: Paolo Bonzini <[email protected]>

Author: bonzini

mesonbuild/cargo/interpreter.py

@@ -19,6 +19,7 @@
 
 from . import builder, version, cfg
 from .toml import load_toml, TomlImplementationMissing
+from .manifest import fixup_meson_varname, CargoLock
 from ..mesonlib import MesonException, MachineChoice
 from .. import coredata, mlog
 from ..wrap.wrap import PackageDefinition
@@ -48,15 +49,6 @@ class DataclassInstance(Protocol):
 )
 
 
-def fixup_meson_varname(name: str) -> str:
-    """Fixup a meson variable name
-
-    :param name: The name to fix
-    :return: the fixed name
-    """
-    return name.replace('-', '_')
-
-
 def _fixup_raw_mappings(d: T.Mapping[str, T.Any], convert_version: bool = True) -> T.MutableMapping[str, T.Any]:
     """Fixup raw cargo mappings to ones more suitable for python to consume.
 
@@ -135,7 +127,7 @@ class Package:
     api: str = dataclasses.field(init=False)
 
     def __post_init__(self) -> None:
-        self.api = _version_to_api(self.version)
+        self.api = version.api(self.version)
 
     @classmethod
     def from_raw(cls, raw: raw.Package) -> Self:
@@ -206,9 +198,9 @@ def __post_init__(self, name: str) -> None:
         api = set()
         for v in self.version:
             if v.startswith(('>=', '==')):
-                api.add(_version_to_api(v[2:].strip()))
+                api.add(version.api(v[2:].strip()))
             elif v.startswith('='):
-                api.add(_version_to_api(v[1:].strip()))
+                api.add(version.api(v[1:].strip()))
         if not api:
             self.api = '0'
         elif len(api) == 1:
@@ -367,18 +359,6 @@ def from_raw(cls, raw: raw.Manifest, path: str = '') -> Self:
         )
 
 
-def _version_to_api(version: str) -> str:
-    # x.y.z -> x
-    # 0.x.y -> 0.x
-    # 0.0.x -> 0
-    vers = version.split('.')
-    if int(vers[0]) != 0:
-        return vers[0]
-    elif len(vers) >= 2 and int(vers[1]) != 0:
-        return f'0.{vers[1]}'
-    return '0'
-
-
 def _dependency_name(package_name: str, api: str, suffix: str = '-rs') -> str:
     basename = package_name[:-len(suffix)] if package_name.endswith(suffix) else package_name
     return f'{basename}-{api}{suffix}'
@@ -805,43 +785,42 @@ def load_wraps(source_dir: str, subproject_dir: str) -> T.List[PackageDefinition
     filename = os.path.join(source_dir, 'Cargo.lock')
     if os.path.exists(filename):
         try:
-            cargolock = T.cast('raw.CargoLock', load_toml(filename))
+            toml = load_toml(filename)
         except TomlImplementationMissing as e:
             mlog.warning('Failed to load Cargo.lock:', str(e), fatal=False)
             return wraps
-        for package in cargolock['package']:
-            name = package['name']
-            version = package['version']
-            subp_name = _dependency_name(name, _version_to_api(version))
-            source = package.get('source')
-            if source is None:
+        raw_cargolock = T.cast('raw.CargoLock', toml)
+        cargolock = CargoLock.from_raw(raw_cargolock)
+        for package in cargolock.package:
+            subp_name = _dependency_name(package.name, version.api(package.version))
+            if package.source is None:
                 # This is project's package, or one of its workspace members.
                 pass
-            elif source == 'registry+https://github.com/rust-lang/crates.io-index':
-                checksum = package.get('checksum')
+            elif package.source == 'registry+https://github.com/rust-lang/crates.io-index':
+                checksum = package.checksum
                 if checksum is None:
-                    checksum = cargolock['metadata'][f'checksum {name} {version} ({source})']
-                url = f'https://crates.io/api/v1/crates/{name}/{version}/download'
-                directory = f'{name}-{version}'
+                    checksum = cargolock.metadata[f'checksum {package.name} {package.version} ({package.source})']
+                url = f'https://crates.io/api/v1/crates/{package.name}/{package.version}/download'
+                directory = f'{package.name}-{package.version}'
                 wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'file', {
                     'directory': directory,
                     'source_url': url,
                     'source_filename': f'{directory}.tar.gz',
                     'source_hash': checksum,
                     'method': 'cargo',
                 }))
-            elif source.startswith('git+'):
-                parts = urllib.parse.urlparse(source[4:])
+            elif package.source.startswith('git+'):
+                parts = urllib.parse.urlparse(package.source[4:])
                 query = urllib.parse.parse_qs(parts.query)
                 branch = query['branch'][0] if 'branch' in query else ''
                 revision = parts.fragment or branch
                 url = urllib.parse.urlunparse(parts._replace(params='', query='', fragment=''))
                 wraps.append(PackageDefinition.from_values(subp_name, subproject_dir, 'git', {
-                    'directory': name,
+                    'directory': package.name,
                     'url': url,
                     'revision': revision,
                     'method': 'cargo',
                 }))
             else:
-                mlog.warning(f'Unsupported source URL in {filename}: {source}')
+                mlog.warning(f'Unsupported source URL in {filename}: {package.source}')
     return wraps

mesonbuild/cargo/manifest.py

@@ -4,3 +4,105 @@
 """Type definitions for cargo manifest files."""
 
 from __future__ import annotations
+
+import dataclasses
+import typing as T
+
+from .. import mlog
+
+if T.TYPE_CHECKING:
+    from typing_extensions import Protocol
+
+    from . import raw
+
+    # Copied from typeshed. Blarg that they don't expose this
+    class DataclassInstance(Protocol):
+        __dataclass_fields__: T.ClassVar[dict[str, dataclasses.Field[T.Any]]]
+
+_DI = T.TypeVar('_DI', bound='DataclassInstance')
+
+_EXTRA_KEYS_WARNING = (
+    "This may (unlikely) be an error in the cargo manifest, or may be a missing "
+    "implementation in Meson. If this issue can be reproduced with the latest "
+    "version of Meson, please help us by opening an issue at "
+    "https://github.com/mesonbuild/meson/issues. Please include the crate and "
+    "version that is generating this warning if possible."
+)
+
+
+def fixup_meson_varname(name: str) -> str:
+    """Fixup a meson variable name
+
+    :param name: The name to fix
+    :return: the fixed name
+    """
+    return name.replace('-', '_')
+
+
+def _raw_to_dataclass(raw: T.Mapping[str, object], cls: T.Type[_DI],
+                      msg: str, **kwargs: T.Callable[[T.Any], object]) -> _DI:
+    """Fixup raw cargo mappings to ones more suitable for python to consume as dataclass.
+
+    * Replaces any `-` with `_` in the keys.
+    * Optionally pass values through the functions in kwargs, in order to do
+      recursive conversions.
+    * Remove and warn on keys that are coming from cargo, but are unknown to
+      our representations.
+
+    This is intended to give users the possibility of things proceeding when a
+    new key is added to Cargo.toml that we don't yet handle, but to still warn
+    them that things might not work.
+
+    :param data: The raw data to look at
+    :param cls: The Dataclass derived type that will be created
+    :param msg: the header for the error message. Usually something like "In N structure".
+    :param convert_version: whether to convert the version field to a Meson compatible one.
+    :return: The original data structure, but with all unknown keys removed.
+    """
+    new_dict = {}
+    unexpected = set()
+    fields = {x.name for x in dataclasses.fields(cls)}
+    for orig_k, v in raw.items():
+        k = fixup_meson_varname(orig_k)
+        if k not in fields:
+            unexpected.add(orig_k)
+            continue
+        if k in kwargs:
+            v = kwargs[k](v)
+        new_dict[k] = v
+
+    if unexpected:
+        mlog.warning(msg, 'has unexpected keys', '"{}".'.format(', '.join(sorted(unexpected))),
+                     _EXTRA_KEYS_WARNING)
+    return cls(**new_dict)
+
+
+@dataclasses.dataclass
+class CargoLockPackage:
+
+    """A description of a package in the Cargo.lock file format."""
+
+    name: str
+    version: str
+    source: T.Optional[str] = None
+    checksum: T.Optional[str] = None
+    dependencies: T.List[str] = dataclasses.field(default_factory=list)
+
+    @classmethod
+    def from_raw(cls, raw: raw.CargoLockPackage) -> CargoLockPackage:
+        return _raw_to_dataclass(raw, cls, 'Cargo.lock package')
+
+
+@dataclasses.dataclass
+class CargoLock:
+
+    """A description of the Cargo.lock file format."""
+
+    version: int = 1
+    package: T.List[CargoLockPackage] = dataclasses.field(default_factory=list)
+    metadata: T.Dict[str, str] = dataclasses.field(default_factory=dict)
+
+    @classmethod
+    def from_raw(cls, raw: raw.CargoLock) -> CargoLock:
+        return _raw_to_dataclass(raw, cls, 'Cargo.lock',
+                                 package=lambda x: [CargoLockPackage.from_raw(p) for p in x])

mesonbuild/cargo/version.py

@@ -7,6 +7,18 @@
 import typing as T
 
 
+def api(version: str) -> str:
+    # x.y.z -> x
+    # 0.x.y -> 0.x
+    # 0.0.x -> 0
+    vers = version.split('.')
+    if int(vers[0]) != 0:
+        return vers[0]
+    elif len(vers) >= 2 and int(vers[1]) != 0:
+        return f'0.{vers[1]}'
+    return '0'
+
+
 def convert(cargo_ver: str) -> T.List[str]:
     """Convert a Cargo compatible version into a Meson compatible one.