diff --git a/dfetch/commands/common.py b/dfetch/commands/common.py index d73f88a8..5ccd2b58 100644 --- a/dfetch/commands/common.py +++ b/dfetch/commands/common.py @@ -5,7 +5,8 @@ import yaml from dfetch.log import get_logger -from dfetch.manifest.manifest import Manifest, get_childmanifests +from dfetch.manifest.manifest import Manifest +from dfetch.manifest.parse import get_childmanifests from dfetch.manifest.project import ProjectEntry logger = get_logger(__name__) diff --git a/dfetch/commands/update.py b/dfetch/commands/update.py index 5f5656f8..481d519f 100644 --- a/dfetch/commands/update.py +++ b/dfetch/commands/update.py @@ -32,11 +32,8 @@ from pathlib import Path import dfetch.commands.command -import dfetch.manifest.manifest import dfetch.manifest.project -import dfetch.manifest.validate -import dfetch.project.git -import dfetch.project.svn +import dfetch.project from dfetch.commands.common import check_child_manifests from dfetch.log import get_logger from dfetch.project.superproject import SuperProject diff --git a/dfetch/commands/validate.py b/dfetch/commands/validate.py index 19295ff1..7b3a992a 100644 --- a/dfetch/commands/validate.py +++ b/dfetch/commands/validate.py @@ -11,8 +11,7 @@ import dfetch.commands.command from dfetch.log import get_logger -from dfetch.manifest.manifest import find_manifest -from dfetch.manifest.validate import validate +from dfetch.manifest.parse import find_manifest, parse logger = get_logger(__name__) @@ -34,6 +33,6 @@ def __call__(self, args: argparse.Namespace) -> None: del args # unused manifest_path = find_manifest() - validate(manifest_path) + parse(manifest_path) manifest_path = os.path.relpath(manifest_path, os.getcwd()) logger.print_info_line(manifest_path, "valid") diff --git a/dfetch/manifest/manifest.py b/dfetch/manifest/manifest.py index 1e329cb7..089adb92 100644 --- a/dfetch/manifest/manifest.py +++ b/dfetch/manifest/manifest.py @@ -21,21 +21,17 @@ import difflib import io import os -import pathlib import re from collections.abc import Sequence from dataclasses import dataclass from typing import IO, Any, Optional, Union import yaml -from typing_extensions import TypedDict +from typing_extensions import NotRequired, TypedDict -from dfetch import DEFAULT_MANIFEST_NAME from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry, ProjectEntryDict from dfetch.manifest.remote import Remote, RemoteDict -from dfetch.manifest.validate import validate -from dfetch.util.util import find_file, prefix_runtime_exceptions logger = get_logger(__name__) @@ -96,13 +92,11 @@ def _guess_project(self, names: Sequence[str]) -> Sequence[str]: ] -class ManifestDict( # pylint: disable=too-many-ancestors - TypedDict, total=False -): # When https://www.python.org/dev/peps/pep-0655/ is accepted, only make remotes optional +class ManifestDict(TypedDict, total=True): # pylint: disable=too-many-ancestors """Serialized dict types.""" version: Union[int, str] - remotes: Sequence[Union[RemoteDict, Remote]] + remotes: NotRequired[Sequence[Union[RemoteDict, Remote]]] projects: Sequence[Union[ProjectEntryDict, ProjectEntry, dict[str, str]]] @@ -362,39 +356,6 @@ def find_name_in_manifest(self, name: str) -> ManifestEntryLocation: raise RuntimeError(f"{name} was not found in the manifest!") -def find_manifest() -> str: - """Find a manifest.""" - paths = find_file(DEFAULT_MANIFEST_NAME, ".") - - if len(paths) == 0: - raise RuntimeError("No manifests were found!") - if len(paths) != 1: - logger.warning( - f"Multiple manifests found, using {pathlib.Path(paths[0]).as_posix()}" - ) - - return os.path.realpath(paths[0]) - - -def get_childmanifests(skip: Optional[list[str]] = None) -> list[Manifest]: - """Get manifest and its path.""" - skip = skip or [] - logger.debug("Looking for sub-manifests") - - childmanifests: list[Manifest] = [] - for path in find_file(DEFAULT_MANIFEST_NAME, "."): - path = os.path.realpath(path) - if path not in skip: - logger.debug(f"Found sub-manifests {path}") - with prefix_runtime_exceptions( - pathlib.Path(path).relative_to(os.path.dirname(os.getcwd())).as_posix() - ): - validate(path) - childmanifests += [Manifest.from_file(path)] - - return childmanifests - - class ManifestDumper(yaml.SafeDumper): # pylint: disable=too-many-ancestors """Dump a manifest YAML. diff --git a/dfetch/manifest/parse.py b/dfetch/manifest/parse.py new file mode 100644 index 00000000..5685a99c --- /dev/null +++ b/dfetch/manifest/parse.py @@ -0,0 +1,106 @@ +"""Validate manifests using StrictYAML.""" + +import os +import pathlib +from typing import Any, Optional, cast + +from strictyaml import StrictYAMLError, YAMLValidationError, load + +from dfetch import DEFAULT_MANIFEST_NAME +from dfetch.log import get_logger +from dfetch.manifest.manifest import Manifest, ManifestDict +from dfetch.manifest.schema import MANIFEST_SCHEMA +from dfetch.util.util import find_file, prefix_runtime_exceptions + +logger = get_logger(__name__) + + +def _ensure_unique(seq: list[dict[str, Any]], key: str, context: str) -> None: + """Ensure values for `key` are unique within a sequence of dicts.""" + values = [item.get(key) for item in seq if key in item] + seen: set[Any] = set() + dups: set[Any] = set() + for val in values: + if val in seen: + dups.add(val) + else: + seen.add(val) + + if dups: + dup_list = ", ".join(sorted(map(str, dups))) + raise RuntimeError( + f"Schema validation failed:\nDuplicate {context}.{key} value(s): {dup_list}" + ) + + +def parse(path: str) -> Manifest: + """Parse & validate the given manifest file against the StrictYAML schema. + + Raises: + RuntimeError: if the file is not valid YAML or violates the schema/uniqueness constraints. + """ + try: + manifest_text = pathlib.Path(path).read_text(encoding="UTF-8") + loaded_manifest = load(manifest_text, schema=MANIFEST_SCHEMA) + except (YAMLValidationError, StrictYAMLError) as err: + raise RuntimeError( + "\n".join( + [ + "Schema validation failed:", + "", + err.context_mark.get_snippet(), + "", + err.problem, + ] + ) + ) from err + + data: dict[str, Any] = cast(dict[str, Any], loaded_manifest.data) + manifest: ManifestDict = data["manifest"] # required + + remotes = manifest.get("remotes", []) # optional + projects = manifest["projects"] # required + + _ensure_unique(remotes, "name", "manifest.remotes") # type: ignore + _ensure_unique(projects, "name", "manifest.projects") # type: ignore + _ensure_unique(projects, "dst", "manifest.projects") # type: ignore + + return Manifest(manifest, text=manifest_text, path=path) + + +def find_manifest() -> str: + """Find a manifest.""" + paths = find_file(DEFAULT_MANIFEST_NAME, ".") + + if len(paths) == 0: + raise RuntimeError("No manifests were found!") + if len(paths) != 1: + logger.warning( + f"Multiple manifests found, using {pathlib.Path(paths[0]).as_posix()}" + ) + + return os.path.realpath(paths[0]) + + +def get_childmanifests(skip: Optional[list[str]] = None) -> list[Manifest]: + """Parse & validate any manifest file in cwd and return a list of all valid manifests.""" + skip = skip or [] + logger.debug("Looking for sub-manifests") + + childmanifests: list[Manifest] = [] + root_dir = os.getcwd() + for path in find_file(DEFAULT_MANIFEST_NAME, root_dir): + path = os.path.realpath(path) + + if os.path.commonprefix((path, root_dir)) != root_dir: + logger.warning(f"Sub-manifest {path} is outside {root_dir}") + continue + + if path not in skip: + logger.debug(f"Found sub-manifests {path}") + with prefix_runtime_exceptions( + pathlib.Path(path).relative_to(os.path.dirname(os.getcwd())).as_posix() + ): + childmanifests += [parse(path)] + + return childmanifests diff --git a/dfetch/manifest/project.py b/dfetch/manifest/project.py index fcd882af..a5eea093 100644 --- a/dfetch/manifest/project.py +++ b/dfetch/manifest/project.py @@ -274,7 +274,7 @@ from collections.abc import Sequence from typing import Optional, Union -from typing_extensions import TypedDict +from typing_extensions import Required, TypedDict from dfetch.manifest.remote import Remote from dfetch.manifest.version import Version @@ -282,7 +282,7 @@ ProjectEntryDict = TypedDict( "ProjectEntryDict", { - "name": str, + "name": Required[str], "revision": str, "remote": str, "src": str, @@ -337,7 +337,9 @@ def from_yaml( Returns: ProjectEntry: An immutable ProjectEntry """ - kwargs: ProjectEntryDict = {} + kwargs: ProjectEntryDict = ( + {} # type: ignore # the Required name key is checked in __init__ + ) for key in ProjectEntryDict.__annotations__.keys(): # pylint: disable=no-member try: kwargs[str(key)] = yamldata[key] # type: ignore diff --git a/dfetch/manifest/validate.py b/dfetch/manifest/validate.py deleted file mode 100644 index 7076031c..00000000 --- a/dfetch/manifest/validate.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Validate manifests using StrictYAML.""" - -import pathlib -from collections.abc import Mapping -from typing import Any, cast - -from strictyaml import StrictYAMLError, YAMLValidationError, load - -from dfetch.manifest.schema import MANIFEST_SCHEMA - - -def _ensure_unique(seq: list[dict[str, Any]], key: str, context: str) -> None: - """Ensure values for `key` are unique within a sequence of dicts.""" - values = [item.get(key) for item in seq if key in item] - seen: set[Any] = set() - dups: set[Any] = set() - for val in values: - if val in seen: - dups.add(val) - else: - seen.add(val) - - if dups: - dup_list = ", ".join(sorted(map(str, dups))) - raise RuntimeError( - f"Schema validation failed:\nDuplicate {context}.{key} value(s): {dup_list}" - ) - - -def validate(path: str) -> None: - """Validate the given manifest file against the StrictYAML schema. - - Raises: - RuntimeError: if the file is not valid YAML or violates the schema/uniqueness constraints. - """ - try: - - loaded_manifest = load( - pathlib.Path(path).read_text(encoding="UTF-8"), schema=MANIFEST_SCHEMA - ) - except (YAMLValidationError, StrictYAMLError) as err: - raise RuntimeError( - "\n".join( - [ - "Schema validation failed:", - "", - err.context_mark.get_snippet(), - "", - err.problem, - ] - ) - ) from err - - data: dict[str, Any] = cast(dict[str, Any], loaded_manifest.data) - manifest: Mapping[str, Any] = data["manifest"] # required - projects: list[dict[str, Any]] = manifest["projects"] # required - remotes: list[dict[str, Any]] = manifest.get("remotes", []) or [] # optional - - _ensure_unique(remotes, "name", "manifest.remotes") - _ensure_unique(projects, "name", "manifest.projects") - _ensure_unique(projects, "dst", "manifest.projects") diff --git a/dfetch/project/superproject.py b/dfetch/project/superproject.py index 53eaab22..5b4afb84 100644 --- a/dfetch/project/superproject.py +++ b/dfetch/project/superproject.py @@ -13,9 +13,9 @@ from collections.abc import Sequence from dfetch.log import get_logger -from dfetch.manifest.manifest import Manifest, find_manifest +from dfetch.manifest.manifest import Manifest +from dfetch.manifest.parse import find_manifest, parse from dfetch.manifest.project import ProjectEntry -from dfetch.manifest.validate import validate from dfetch.project.git import GitSubProject from dfetch.project.subproject import SubProject from dfetch.project.svn import SvnSubProject @@ -37,10 +37,9 @@ def __init__(self) -> None: """Create a SuperProject by looking for a manifest file.""" logger.debug("Looking for manifest") manifest_path = find_manifest() - validate(manifest_path) logger.debug(f"Using manifest {manifest_path}") - self._manifest = Manifest.from_file(manifest_path) + self._manifest = parse(manifest_path) self._root_directory = os.path.dirname(self._manifest.path) @property diff --git a/pyproject.toml b/pyproject.toml index 9ac6def5..57c5933c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -189,8 +189,8 @@ skip = "*.cast,./venv,**/plantuml-c4/**,./example,.mypy_cache,./doc/_build/**,./ "features/steps/*" = ["F811"] [tool.pyright] -exclude = ["doc/static/uml/generate_diagram.py", "./doc/_ext/sphinxcontrib_asciinema/", "./doc/_build"] -standard = ["dfetch", "features"] +exclude = ["doc/static/uml/generate_diagram.py", "./doc/_ext/sphinxcontrib_asciinema/", "./doc/_build", "**/node_modules", "**/__pycache__", "**/.*", "venv"] +# standard = ["dfetch", "features"] reportMissingImports = false reportMissingModuleSource = false pythonVersion = "3.9" diff --git a/script/create_sbom.py b/script/create_sbom.py index 0bc9bafa..9a4bcc31 100644 --- a/script/create_sbom.py +++ b/script/create_sbom.py @@ -15,13 +15,18 @@ DEPS = f"{PROJECT_DIR}[sbom]" - -PLATFORM_NAME = "nix" - -if sys.platform.startswith("darwin"): - PLATFORM_NAME = "osx" -elif sys.platform.startswith("win"): - PLATFORM_NAME = "win" +PLATFORM_MAPPING = { + "darwin": "osx", + "win": "win", +} +PLATFORM_NAME = next( + ( + name + for prefix, name in PLATFORM_MAPPING.items() + if sys.platform.startswith(prefix) + ), + "nix", +) @contextlib.contextmanager diff --git a/script/package.py b/script/package.py index bd82d296..65c7f32a 100644 --- a/script/package.py +++ b/script/package.py @@ -46,12 +46,18 @@ WINDOWS_ICO_PATH = Path(WINDOWS_ICO).resolve() if WINDOWS_ICO else None -PLATFORM_NAME = "nix" - -if sys.platform.startswith("darwin"): - PLATFORM_NAME = "osx" -elif sys.platform.startswith("win"): - PLATFORM_NAME = "win" +PLATFORM_MAPPING = { + "darwin": "osx", + "win": "win", +} +PLATFORM_NAME = next( + ( + name + for prefix, name in PLATFORM_MAPPING.items() + if sys.platform.startswith(prefix) + ), + "nix", +) def run_command(command: list[str]) -> None: diff --git a/tests/test_check.py b/tests/test_check.py index d67a0559..bf9080ef 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -34,7 +34,7 @@ def test_check(name, projects): with patch("dfetch.commands.check.SuperProject", return_value=fake_superproject): with patch( - "dfetch.manifest.manifest.get_childmanifests" + "dfetch.manifest.parse.get_childmanifests" ) as mocked_get_childmanifests: with patch("dfetch.project.make") as mocked_make: with patch("os.path.exists"): diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 4726fd7f..2983b987 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -14,9 +14,8 @@ ManifestDict, ManifestEntryLocation, RequestedProjectNotFoundError, - find_manifest, - get_childmanifests, ) +from dfetch.manifest.parse import find_manifest, get_childmanifests from dfetch.manifest.project import ProjectEntry BASIC_MANIFEST = """ @@ -99,20 +98,20 @@ def test_construct_from_dict() -> None: def test_no_manifests_found() -> None: - with patch("dfetch.manifest.manifest.find_file"): + with patch("dfetch.manifest.parse.find_file"): with pytest.raises(RuntimeError): find_manifest() def test_multiple_manifests_found() -> None: - with patch("dfetch.manifest.manifest.find_file") as find_file_mock: + with patch("dfetch.manifest.parse.find_file") as find_file_mock: find_file_mock.return_value = [DEFAULT_MANIFEST_NAME, "manifest2.yaml"] assert os.path.realpath(DEFAULT_MANIFEST_NAME) == find_manifest() def test_single_manifest_found() -> None: - with patch("dfetch.manifest.manifest.find_file") as find_file_mock: + with patch("dfetch.manifest.parse.find_file") as find_file_mock: find_file_mock.return_value = [DEFAULT_MANIFEST_NAME] joined = os.path.realpath(os.path.join(os.getcwd(), DEFAULT_MANIFEST_NAME)) @@ -140,20 +139,19 @@ def test_single_manifest_found() -> None: def test_get_childmanifests(name, manifest_paths) -> None: parent = ProjectEntry({"name": "name"}) - with patch("dfetch.manifest.manifest.find_file") as find_file_mock: - with patch("dfetch.manifest.manifest.validate"): - with patch("dfetch.manifest.manifest.Manifest") as manifest_mock: - find_file_mock.return_value = manifest_paths + with patch("dfetch.manifest.parse.find_file") as find_file_mock: + with patch("dfetch.manifest.parse.parse") as parse_mock: + find_file_mock.return_value = manifest_paths - found_childmanifests = get_childmanifests([parent.name]) + found_childmanifests = get_childmanifests([parent.name]) - assert len(found_childmanifests) == len(manifest_paths) + assert len(found_childmanifests) == len(manifest_paths) - for path, call in zip( - manifest_paths, - manifest_mock.from_file.call_args_list, # , strict=True - ): - assert os.path.realpath(path) == call[0][0] + for path, call in zip( + manifest_paths, + parse_mock.call_args_list, # , strict=True + ): + assert os.path.realpath(path) == call[0][0] def test_suggestion_found() -> None: diff --git a/tests/test_update.py b/tests/test_update.py index 47dd58fe..24b9169e 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -34,7 +34,7 @@ def test_update(name, projects): with patch("dfetch.commands.update.SuperProject", return_value=fake_superproject): with patch( - "dfetch.manifest.manifest.get_childmanifests" + "dfetch.manifest.parse.get_childmanifests" ) as mocked_get_childmanifests: with patch("dfetch.project.make") as mocked_make: with patch("os.path.exists"): @@ -58,7 +58,7 @@ def test_forced_update(): with patch("dfetch.commands.update.SuperProject", return_value=fake_superproject): with patch( - "dfetch.manifest.manifest.get_childmanifests" + "dfetch.manifest.parse.get_childmanifests" ) as mocked_get_childmanifests: with patch("dfetch.project.make") as mocked_make: with patch("os.path.exists"):