feat: add template dynamic-metadata plugin and support requesting other fields (#1047)

Implementation of
scikit-build/dynamic-metadata#21. Mentioned in
https://discuss.python.org/t/partially-dynamic-project-metadata-proposal-pre-pep/88608.

~~TODO: support specifying a non-dynamic field. It should not be an
error if it exists already!~~

---------

Signed-off-by: Henry Schreiner <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: henryiii

docs/api/scikit_build_core.metadata.rst

@@ -32,3 +32,11 @@ scikit\_build\_core.metadata.setuptools\_scm module
    :members:
    :show-inheritance:
    :undoc-members:
+
+scikit\_build\_core.metadata.template module
+--------------------------------------------
+
+.. automodule:: scikit_build_core.metadata.template
+   :members:
+   :show-inheritance:
+   :undoc-members:

docs/configuration/dynamic.md

@@ -52,7 +52,7 @@ dynamic_version()
 project(MyPackage VERSION ${PROJECT_VERSION})
 ```
 
-## `version`: Regex
+## Regex
 
 If you want to pull a string-valued expression (usually version) from an
 existing file, you can the integrated `regex` plugin to pull the information.
@@ -112,6 +112,30 @@ metadata.readme.provider = "scikit_build_core.metadata.fancy_pypi_readme"
 # tool.hatch.metadata.hooks.fancy-pypi-readme options here
 ```
 
+```{versionchanged} 0.11.2
+
+The version number feature now works.
+```
+
+## Template
+
+You can access other metadata fields and produce templated outputs.
+
+```toml
+[tool.scikit-build.metadata.optional-dependencies]
+provider = "scikit_build_core.metadata.template"
+result = {"dev" = ["{project[name]}=={project[version]}"]}
+```
+
+You can use `project` to access the current metadata values. You can reference
+other dynamic metadata fields, and they will be computed before this one. You
+can use `result` to specify the output. The result must match the type of the
+metadata field you are writing to.
+
+```{versionadded} 0.11.2
+
+```
+
 ## `build-system.requires`: Scikit-build-core's `build.requires`
 
 If you need to inject and manipulate additional `build-system.requires`, you can

src/scikit_build_core/build/metadata.py

@@ -13,7 +13,7 @@
     extras_build_system,
     extras_top_level,
 )
-from ..settings._load_provider import load_dynamic_metadata
+from ..builder._load_provider import process_dynamic_metadata
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
@@ -34,23 +34,16 @@ def __dir__() -> list[str]:
     errors.ExceptionGroup = ExceptionGroup  # type: ignore[misc, assignment]
 
 
-# If pyproject-metadata eventually supports updates, this can be simplified
 def get_standard_metadata(
     pyproject_dict: Mapping[str, Any],
     settings: ScikitBuildSettings,
 ) -> StandardMetadata:
-    new_pyproject_dict = copy.deepcopy(pyproject_dict)
+    new_pyproject_dict = copy.deepcopy(dict(pyproject_dict))
 
     # Handle any dynamic metadata
-    for field, provider, config in load_dynamic_metadata(settings.metadata):
-        if provider is None:
-            msg = f"{field} is missing provider"
-            raise KeyError(msg)
-        if field not in pyproject_dict.get("project", {}).get("dynamic", []):
-            msg = f"{field} is not in project.dynamic"
-            raise KeyError(msg)
-        new_pyproject_dict["project"][field] = provider.dynamic_metadata(field, config)
-        new_pyproject_dict["project"]["dynamic"].remove(field)
+    new_pyproject_dict["project"] = process_dynamic_metadata(
+        new_pyproject_dict["project"], settings.metadata
+    )
 
     if settings.strict_config:
         extra_keys_top = extras_top_level(new_pyproject_dict)

src/scikit_build_core/builder/_load_provider.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import dataclasses
+import importlib
+import inspect
+import sys
+from collections.abc import Iterator, Mapping
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Protocol, Union, runtime_checkable
+
+if TYPE_CHECKING:
+    from collections.abc import Generator, Iterable
+
+    StrMapping = Mapping[str, Any]
+else:
+    StrMapping = Mapping
+
+from ..metadata import _ALL_FIELDS
+
+__all__ = ["load_provider", "process_dynamic_metadata"]
+
+
+def __dir__() -> list[str]:
+    return __all__
+
+
+@runtime_checkable
+class DynamicMetadataProtocol(Protocol):
+    def dynamic_metadata(
+        self,
+        fields: Iterable[str],
+        settings: dict[str, Any],
+        project: Mapping[str, Any],
+    ) -> dict[str, Any]: ...
+
+
+@runtime_checkable
+class DynamicMetadataRequirementsProtocol(DynamicMetadataProtocol, Protocol):
+    def get_requires_for_dynamic_metadata(
+        self, settings: dict[str, Any]
+    ) -> list[str]: ...
+
+
+@runtime_checkable
+class DynamicMetadataWheelProtocol(DynamicMetadataProtocol, Protocol):
+    def dynamic_wheel(self, field: str, settings: Mapping[str, Any]) -> bool: ...
+
+
+DMProtocols = Union[
+    DynamicMetadataProtocol,
+    DynamicMetadataRequirementsProtocol,
+    DynamicMetadataWheelProtocol,
+]
+
+
+def load_provider(
+    provider: str,
+    provider_path: str | None = None,
+) -> DMProtocols:
+    if provider_path is None:
+        return importlib.import_module(provider)
+
+    if not Path(provider_path).is_dir():
+        msg = "provider-path must be an existing directory"
+        raise AssertionError(msg)
+
+    try:
+        sys.path.insert(0, provider_path)
+        return importlib.import_module(provider)
+    finally:
+        sys.path.pop(0)
+
+
+def _load_dynamic_metadata(
+    metadata: Mapping[str, Mapping[str, str]],
+) -> Generator[tuple[str, DMProtocols, dict[str, Any]], None, None]:
+    for field, orig_config in metadata.items():
+        if "provider" not in orig_config:
+            msg = "Missing provider in dynamic metadata"
+            raise KeyError(msg)
+
+        if field not in _ALL_FIELDS:
+            msg = f"{field} is not a valid field"
+            raise KeyError(msg)
+        config = dict(orig_config)
+        provider = config.pop("provider")
+        provider_path = config.pop("provider-path", None)
+        loaded_provider = load_provider(provider, provider_path)
+        yield field, loaded_provider, config
+
+
+@dataclasses.dataclass
+class DynamicPyProject(StrMapping):
+    settings: dict[str, dict[str, Any]]
+    project: dict[str, Any]
+    providers: dict[str, DMProtocols]
+
+    def __getitem__(self, key: str) -> Any:
+        # Try to get the settings from either the static file or dynamic metadata provider
+        if key in self.project:
+            return self.project[key]
+
+        # Check if we are in a loop, i.e. something else is already requesting
+        # this key while trying to get another key
+        if key not in self.providers:
+            dep_type = "missing" if key in self.settings else "circular"
+            msg = f"Encountered a {dep_type} dependency at {key}"
+            raise ValueError(msg)
+
+        provider = self.providers.pop(key)
+        sig = inspect.signature(provider.dynamic_metadata)
+        if len(sig.parameters) < 3:
+            # Backcompat for dynamic_metadata without metadata dict
+            self.project[key] = provider.dynamic_metadata(  # type: ignore[call-arg]
+                key, self.settings[key]
+            )
+        else:
+            self.project[key] = provider.dynamic_metadata(key, self.settings[key], self)
+        self.project["dynamic"].remove(key)
+
+        return self.project[key]
+
+    def __iter__(self) -> Iterator[str]:
+        # Iterate over the keys of the static settings
+        yield from self.project
+
+        # Iterate over the keys of the dynamic metadata providers
+        yield from self.providers
+
+    def __len__(self) -> int:
+        return len(self.project) + len(self.providers)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self.project or key in self.providers
+
+
+def process_dynamic_metadata(
+    project: Mapping[str, Any],
+    metadata: Mapping[str, Mapping[str, Any]],
+) -> dict[str, Any]:
+    initial = {f: (p, c) for (f, p, c) in _load_dynamic_metadata(metadata)}
+
+    settings = DynamicPyProject(
+        settings={f: c for f, (_, c) in initial.items()},
+        project=dict(project),
+        providers={k: v for k, (v, _) in initial.items()},
+    )
+
+    return dict(settings)

src/scikit_build_core/builder/get_requires.py

@@ -19,8 +19,8 @@
     get_ninja_programs,
 )
 from ..resources import resources
-from ..settings._load_provider import load_provider
 from ..settings.skbuild_read_settings import SettingsReader
+from ._load_provider import load_provider
 
 if TYPE_CHECKING:
     from collections.abc import Generator, Mapping

src/scikit_build_core/metadata/__init__.py

@@ -1,3 +1,87 @@
 from __future__ import annotations
 
-__all__: list[str] = []
+import typing
+
+if typing.TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+__all__: list[str] = ["_ALL_FIELDS", "_process_dynamic_metadata"]
+
+
+# Name is not dynamically settable, so not in this list
+_STR_FIELDS = frozenset(
+    [
+        "version",
+        "description",
+        "requires-python",
+        "license",
+    ]
+)
+
+# Dynamic is not dynamically settable, so not in this list
+_LIST_STR_FIELDS = frozenset(
+    [
+        "classifiers",
+        "keywords",
+        "dependencies",
+        "license_files",
+    ]
+)
+
+_DICT_STR_FIELDS = frozenset(
+    [
+        "urls",
+        "authors",
+        "maintainers",
+    ]
+)
+
+# "dynamic" and "name" can't be set or requested
+_ALL_FIELDS = (
+    _STR_FIELDS
+    | _LIST_STR_FIELDS
+    | _DICT_STR_FIELDS
+    | frozenset(
+        [
+            "optional-dependencies",
+            "readme",
+        ]
+    )
+)
+
+T = typing.TypeVar("T", bound="str | list[str] | dict[str, str] | dict[str, list[str]]")
+
+
+def _process_dynamic_metadata(field: str, action: Callable[[str], str], result: T) -> T:
+    """
+    Helper function for processing the an action on the various possible metadata fields.
+    """
+
+    if field in _STR_FIELDS:
+        if not isinstance(result, str):
+            msg = f"Field {field!r} must be a string"
+            raise RuntimeError(msg)
+        return action(result)  # type: ignore[return-value]
+    if field in _LIST_STR_FIELDS:
+        if not (isinstance(result, list) and all(isinstance(r, str) for r in result)):
+            msg = f"Field {field!r} must be a list of strings"
+            raise RuntimeError(msg)
+        return [action(r) for r in result]  # type: ignore[return-value]
+    if field in _DICT_STR_FIELDS | {"readme"}:
+        if not isinstance(result, dict) or not all(
+            isinstance(v, str) for v in result.values()
+        ):
+            msg = f"Field {field!r} must be a dictionary of strings"
+            raise RuntimeError(msg)
+        return {k: action(v) for k, v in result.items()}  # type: ignore[return-value]
+    if field == "optional-dependencies":
+        if not isinstance(result, dict) or not all(
+            isinstance(v, list) for v in result.values()
+        ):
+            msg = "Field 'optional-dependencies' must be a dictionary of lists"
+            raise RuntimeError(msg)
+        return {k: [action(r) for r in v] for k, v in result.items()}  # type: ignore[return-value]
+
+    msg = f"Unsupported field {field!r} for action"
+    raise RuntimeError(msg)

src/scikit_build_core/metadata/fancy_pypi_readme.py

@@ -1,10 +1,17 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING, Any
 
 from .._compat import tomllib
 
-__all__ = ["dynamic_metadata", "get_requires_for_dynamic_metadata"]
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+__all__ = [
+    "dynamic_metadata",
+    "get_requires_for_dynamic_metadata",
+]
 
 
 def __dir__() -> list[str]:
@@ -13,7 +20,8 @@ def __dir__() -> list[str]:
 
 def dynamic_metadata(
     field: str,
-    settings: dict[str, list[str] | str] | None = None,
+    settings: dict[str, list[str] | str],
+    project: Mapping[str, Any],
 ) -> str | dict[str, str]:
     from hatch_fancy_pypi_readme._builder import build_text
     from hatch_fancy_pypi_readme._config import load_and_validate_config
@@ -36,7 +44,9 @@ def dynamic_metadata(
     if hasattr(config, "substitutions"):
         try:
             # We don't have access to the version at this point
-            text = build_text(config.fragments, config.substitutions, "")
+            text = build_text(
+                config.fragments, config.substitutions, version=project["version"]
+            )
         except TypeError:
             # Version 23.2.0 and before don't have a version field
             # pylint: disable-next=no-value-for-parameter