Load manifest only once

dfetch/commands/common.py

@@ -5,7 +5,8 @@
 import yaml
 
 from dfetch.log import get_logger
-from dfetch.manifest.manifest import Manifest, get_childmanifests
+from dfetch.manifest.manifest import Manifest
+from dfetch.manifest.parse import get_childmanifests
 from dfetch.manifest.project import ProjectEntry
 
 logger = get_logger(__name__)

dfetch/commands/update.py

@@ -32,11 +32,8 @@
 from pathlib import Path
 
 import dfetch.commands.command
-import dfetch.manifest.manifest
 import dfetch.manifest.project
-import dfetch.manifest.validate
-import dfetch.project.git
-import dfetch.project.svn
+import dfetch.project
 from dfetch.commands.common import check_child_manifests
 from dfetch.log import get_logger
 from dfetch.project.superproject import SuperProject

dfetch/commands/validate.py

@@ -11,8 +11,7 @@
 
 import dfetch.commands.command
 from dfetch.log import get_logger
-from dfetch.manifest.manifest import find_manifest
-from dfetch.manifest.validate import validate
+from dfetch.manifest.parse import find_manifest, parse
 
 logger = get_logger(__name__)
 
@@ -34,6 +33,6 @@ def __call__(self, args: argparse.Namespace) -> None:
         del args  # unused
 
         manifest_path = find_manifest()
-        validate(manifest_path)
+        parse(manifest_path)
         manifest_path = os.path.relpath(manifest_path, os.getcwd())
         logger.print_info_line(manifest_path, "valid")

dfetch/manifest/manifest.py

@@ -21,21 +21,17 @@
 import difflib
 import io
 import os
-import pathlib
 import re
 from collections.abc import Sequence
 from dataclasses import dataclass
 from typing import IO, Any, Optional, Union
 
 import yaml
-from typing_extensions import TypedDict
+from typing_extensions import NotRequired, TypedDict
 
-from dfetch import DEFAULT_MANIFEST_NAME
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry, ProjectEntryDict
 from dfetch.manifest.remote import Remote, RemoteDict
-from dfetch.manifest.validate import validate
-from dfetch.util.util import find_file, prefix_runtime_exceptions
 
 logger = get_logger(__name__)
 
@@ -96,13 +92,11 @@ def _guess_project(self, names: Sequence[str]) -> Sequence[str]:
         ]
 
 
-class ManifestDict(  # pylint: disable=too-many-ancestors
-    TypedDict, total=False
-):  # When https://www.python.org/dev/peps/pep-0655/ is accepted, only make remotes optional
+class ManifestDict(TypedDict, total=True):  # pylint: disable=too-many-ancestors
     """Serialized dict types."""
 
     version: Union[int, str]
-    remotes: Sequence[Union[RemoteDict, Remote]]
+    remotes: NotRequired[Sequence[Union[RemoteDict, Remote]]]
     projects: Sequence[Union[ProjectEntryDict, ProjectEntry, dict[str, str]]]
 
 
@@ -362,39 +356,6 @@ def find_name_in_manifest(self, name: str) -> ManifestEntryLocation:
         raise RuntimeError(f"{name} was not found in the manifest!")
 
 
-def find_manifest() -> str:
-    """Find a manifest."""
-    paths = find_file(DEFAULT_MANIFEST_NAME, ".")
-
-    if len(paths) == 0:
-        raise RuntimeError("No manifests were found!")
-    if len(paths) != 1:
-        logger.warning(
-            f"Multiple manifests found, using {pathlib.Path(paths[0]).as_posix()}"
-        )
-
-    return os.path.realpath(paths[0])
-
-
-def get_childmanifests(skip: Optional[list[str]] = None) -> list[Manifest]:
-    """Get manifest and its path."""
-    skip = skip or []
-    logger.debug("Looking for sub-manifests")
-
-    childmanifests: list[Manifest] = []
-    for path in find_file(DEFAULT_MANIFEST_NAME, "."):
-        path = os.path.realpath(path)
-        if path not in skip:
-            logger.debug(f"Found sub-manifests {path}")
-            with prefix_runtime_exceptions(
-                pathlib.Path(path).relative_to(os.path.dirname(os.getcwd())).as_posix()
-            ):
-                validate(path)
-            childmanifests += [Manifest.from_file(path)]
-
-    return childmanifests
-
-
 class ManifestDumper(yaml.SafeDumper):  # pylint: disable=too-many-ancestors
     """Dump a manifest YAML.
 

dfetch/manifest/parse.py

@@ -0,0 +1,106 @@
+"""Validate manifests using StrictYAML."""
+
+import os
+import pathlib
+from typing import Any, Optional, cast
+
+from strictyaml import StrictYAMLError, YAMLValidationError, load
+
+from dfetch import DEFAULT_MANIFEST_NAME
+from dfetch.log import get_logger
+from dfetch.manifest.manifest import Manifest, ManifestDict
+from dfetch.manifest.schema import MANIFEST_SCHEMA
+from dfetch.util.util import find_file, prefix_runtime_exceptions
+
+logger = get_logger(__name__)
+
+
+def _ensure_unique(seq: list[dict[str, Any]], key: str, context: str) -> None:
+    """Ensure values for `key` are unique within a sequence of dicts."""
+    values = [item.get(key) for item in seq if key in item]
+    seen: set[Any] = set()
+    dups: set[Any] = set()
+    for val in values:
+        if val in seen:
+            dups.add(val)
+        else:
+            seen.add(val)
+
+    if dups:
+        dup_list = ", ".join(sorted(map(str, dups)))
+        raise RuntimeError(
+            f"Schema validation failed:\nDuplicate {context}.{key} value(s): {dup_list}"
+        )
+
+
+def parse(path: str) -> Manifest:
+    """Parse & validate the given manifest file against the StrictYAML schema.
+
+    Raises:
+        RuntimeError: if the file is not valid YAML or violates the schema/uniqueness constraints.
+    """
+    try:
+        manifest_text = pathlib.Path(path).read_text(encoding="UTF-8")
+        loaded_manifest = load(manifest_text, schema=MANIFEST_SCHEMA)
+    except (YAMLValidationError, StrictYAMLError) as err:
+        raise RuntimeError(
+            "\n".join(
+                [
+                    "Schema validation failed:",
+                    "",
+                    err.context_mark.get_snippet(),
+                    "",
+                    err.problem,
+                ]
+            )
+        ) from err
+
+    data: dict[str, Any] = cast(dict[str, Any], loaded_manifest.data)
+    manifest: ManifestDict = data["manifest"]  # required
+
+    remotes = manifest.get("remotes", [])  # optional
+    projects = manifest["projects"]  # required
+
+    _ensure_unique(remotes, "name", "manifest.remotes")  # type: ignore
+    _ensure_unique(projects, "name", "manifest.projects")  # type: ignore
+    _ensure_unique(projects, "dst", "manifest.projects")  # type: ignore
+
+    return Manifest(manifest, text=manifest_text, path=path)
+
+
+def find_manifest() -> str:
+    """Find a manifest."""
+    paths = find_file(DEFAULT_MANIFEST_NAME, ".")
+
+    if len(paths) == 0:
+        raise RuntimeError("No manifests were found!")
+    if len(paths) != 1:
+        logger.warning(
+            f"Multiple manifests found, using {pathlib.Path(paths[0]).as_posix()}"
+        )
+
+    return os.path.realpath(paths[0])
+
+
+def get_childmanifests(skip: Optional[list[str]] = None) -> list[Manifest]:
+    """Parse & validate any manifest file in cwd and return a list of all valid manifests."""
+    skip = skip or []
+    logger.debug("Looking for sub-manifests")
+
+    childmanifests: list[Manifest] = []
+    root_dir = os.getcwd()
+    for path in find_file(DEFAULT_MANIFEST_NAME, root_dir):
+        path = os.path.realpath(path)
+
+        if os.path.commonprefix((path, root_dir)) != root_dir:
+            logger.warning(f"Sub-manifest {path} is outside {root_dir}")
+            continue
+
+        if path not in skip:
+            logger.debug(f"Found sub-manifests {path}")
+            with prefix_runtime_exceptions(
+                pathlib.Path(path).relative_to(os.path.dirname(os.getcwd())).as_posix()
+            ):
+                childmanifests += [parse(path)]
+
+    return childmanifests

dfetch/manifest/project.py

@@ -274,15 +274,15 @@
 from collections.abc import Sequence
 from typing import Optional, Union
 
-from typing_extensions import TypedDict
+from typing_extensions import Required, TypedDict
 
 from dfetch.manifest.remote import Remote
 from dfetch.manifest.version import Version
 
 ProjectEntryDict = TypedDict(
     "ProjectEntryDict",
     {
-        "name": str,
+        "name": Required[str],
         "revision": str,
         "remote": str,
         "src": str,
@@ -337,7 +337,9 @@ def from_yaml(
         Returns:
             ProjectEntry:  An immutable ProjectEntry
         """
-        kwargs: ProjectEntryDict = {}
+        kwargs: ProjectEntryDict = (
+            {}  # type: ignore # the Required name key is checked in __init__
+        )
         for key in ProjectEntryDict.__annotations__.keys():  # pylint: disable=no-member
             try:
                 kwargs[str(key)] = yamldata[key]  # type: ignore

dfetch/project/superproject.py

@@ -13,9 +13,9 @@
 from collections.abc import Sequence
 
 from dfetch.log import get_logger
-from dfetch.manifest.manifest import Manifest, find_manifest
+from dfetch.manifest.manifest import Manifest
+from dfetch.manifest.parse import find_manifest, parse
 from dfetch.manifest.project import ProjectEntry
-from dfetch.manifest.validate import validate
 from dfetch.project.git import GitSubProject
 from dfetch.project.subproject import SubProject
 from dfetch.project.svn import SvnSubProject
@@ -37,10 +37,9 @@ def __init__(self) -> None:
         """Create a SuperProject by looking for a manifest file."""
         logger.debug("Looking for manifest")
         manifest_path = find_manifest()
-        validate(manifest_path)
 
         logger.debug(f"Using manifest {manifest_path}")
-        self._manifest = Manifest.from_file(manifest_path)
+        self._manifest = parse(manifest_path)
         self._root_directory = os.path.dirname(self._manifest.path)
 
     @property

pyproject.toml

@@ -189,8 +189,8 @@ skip = "*.cast,./venv,**/plantuml-c4/**,./example,.mypy_cache,./doc/_build/**,./
 "features/steps/*" = ["F811"]
 
 [tool.pyright]
-exclude = ["doc/static/uml/generate_diagram.py", "./doc/_ext/sphinxcontrib_asciinema/", "./doc/_build"]
-standard = ["dfetch", "features"]
+exclude = ["doc/static/uml/generate_diagram.py", "./doc/_ext/sphinxcontrib_asciinema/", "./doc/_build", "**/node_modules", "**/__pycache__", "**/.*", "venv"]
+# standard = ["dfetch", "features"]
 reportMissingImports = false
 reportMissingModuleSource = false
 pythonVersion = "3.9"

script/create_sbom.py

@@ -15,13 +15,18 @@
 
 DEPS = f"{PROJECT_DIR}[sbom]"
 
-
-PLATFORM_NAME = "nix"
-
-if sys.platform.startswith("darwin"):
-    PLATFORM_NAME = "osx"
-elif sys.platform.startswith("win"):
-    PLATFORM_NAME = "win"
+PLATFORM_MAPPING = {
+    "darwin": "osx",
+    "win": "win",
+}
+PLATFORM_NAME = next(
+    (
+        name
+        for prefix, name in PLATFORM_MAPPING.items()
+        if sys.platform.startswith(prefix)
+    ),
+    "nix",
+)
 
 
 @contextlib.contextmanager