diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 6235100f27..07fb7b9ee4 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -3,6 +3,7 @@ addopts = --capture=no --code-highlight=yes --color=yes + -rA --strict-config --strict-markers --verbose @@ -18,3 +19,4 @@ markers = clp: mark tests that use the CLP storage engine clp_s: mark tests that use the CLP-S storage engine core: mark tests that test the CLP core binaries + package: mark tests that run when the CLP package is active diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml index dd0cf797fa..315336e2b0 100644 --- a/integration-tests/pyproject.toml +++ b/integration-tests/pyproject.toml @@ -27,6 +27,8 @@ dev = [ "ruff>=0.11.12", "pytest>=8.4.1", "pytest-env>=1.1.5", + "PyYAML>=6.0", + "types-PyYAML>=6.0.12.20240808", ] [tool.mypy] diff --git a/integration-tests/tests/conftest.py b/integration-tests/tests/conftest.py index 2cb709a249..1e475b1bd7 100644 --- a/integration-tests/tests/conftest.py +++ b/integration-tests/tests/conftest.py @@ -3,4 +3,6 @@ pytest_plugins = [ "tests.fixtures.integration_test_config", "tests.fixtures.integration_test_logs", + "tests.fixtures.package_instance_fixtures", + "tests.fixtures.package_config_fixtures", ] diff --git a/integration-tests/tests/fixtures/integration_test_config.py b/integration-tests/tests/fixtures/integration_test_config.py index 3d53203c31..7bca44fe05 100644 --- a/integration-tests/tests/fixtures/integration_test_config.py +++ b/integration-tests/tests/fixtures/integration_test_config.py @@ -7,7 +7,6 @@ from tests.utils.config import ( CoreConfig, IntegrationTestConfig, - PackageConfig, ) from tests.utils.utils import get_env_var @@ -20,14 +19,6 @@ def core_config() -> CoreConfig: ) -@pytest.fixture(scope="session") -def package_config() -> PackageConfig: - """Fixture that provides a PackageConfig shared across tests.""" - return PackageConfig( - clp_package_dir=Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve() - ) - - @pytest.fixture(scope="session") def integration_test_config() -> IntegrationTestConfig: """Fixture that provides an IntegrationTestConfig shared across tests.""" diff --git a/integration-tests/tests/fixtures/package_config_fixtures.py b/integration-tests/tests/fixtures/package_config_fixtures.py new file mode 100644 index 0000000000..f5c0dfe5f8 --- /dev/null +++ b/integration-tests/tests/fixtures/package_config_fixtures.py @@ -0,0 +1,56 @@ +"""Fixtures that create and remove temporary config files for CLP packages.""" + +import logging +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS +from tests.utils.config import PackageConfig +from tests.utils.utils import get_env_var + +logger = logging.getLogger(__name__) + + +def _build_package_config_for_mode(mode_name: str) -> PackageConfig: + """Construct a PackageConfig for the given `mode_name`.""" + if mode_name not in CLP_MODE_CONFIGS: + err_msg = f"Unknown CLP mode '{mode_name}'." + raise KeyError(err_msg) + + clp_package_dir = Path(get_env_var("CLP_PACKAGE_DIR")).expanduser().resolve() + test_root_dir = Path(get_env_var("CLP_BUILD_DIR")).expanduser().resolve() / "integration-tests" + build_config = CLP_MODE_CONFIGS[mode_name][0] + required_components = CLP_MODE_CONFIGS[mode_name][1] + + return PackageConfig( + clp_package_dir=clp_package_dir, + test_root_dir=test_root_dir, + mode_name=mode_name, + build_config=build_config, + component_list=required_components, + ) + + +@pytest.fixture +def clp_config( + request: pytest.FixtureRequest, +) -> Iterator[PackageConfig]: + """ + Parameterized fixture that creates and maintains a PackageConfig object for a specific mode of + operation. + """ + mode_name: str = request.param + logger.info("Creating a temporary config file for the %s package...", mode_name) + + package_config = _build_package_config_for_mode(mode_name) + + logger.info("The temporary config file has been written for the %s package.", mode_name) + + try: + yield package_config + finally: + logger.info("Removing the temporary config file...") + package_config.temp_config_file_path.unlink() + logger.info("The temporary config file has been removed.") diff --git a/integration-tests/tests/fixtures/package_instance_fixtures.py b/integration-tests/tests/fixtures/package_instance_fixtures.py new file mode 100644 index 0000000000..582eb3b5df --- /dev/null +++ b/integration-tests/tests/fixtures/package_instance_fixtures.py @@ -0,0 +1,50 @@ +"""Fixtures that start and stop CLP package instances for integration tests.""" + +import logging +import subprocess +from collections.abc import Iterator + +import pytest + +from tests.utils.config import ( + PackageConfig, + PackageInstance, +) +from tests.utils.package_utils import ( + start_clp_package, + stop_clp_package, +) + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def clp_package( + clp_config: PackageConfig, +) -> Iterator[PackageInstance]: + """ + Parameterized fixture that starts a instance of the CLP package in the configuration described + in PackageConfig, and gracefully stops it at teardown. + """ + mode_name = clp_config.mode_name + logger.info("Starting up the %s package...", mode_name) + + start_clp_package(clp_config) + + try: + instance = PackageInstance(package_config=clp_config) + instance_id = instance.clp_instance_id + logger.info( + "An instance of the %s package was started successfully. Its instance ID is '%s'", + mode_name, + instance_id, + ) + yield instance + finally: + logger.info("Now stopping the %s package...", mode_name) + if instance is not None: + stop_clp_package(instance) + else: + # This means setup failed after start; fall back to calling stop script directly + subprocess.run([str(clp_config.stop_script_path)], check=True) + logger.info("The %s package was stopped successfully.", mode_name) diff --git a/integration-tests/tests/test_package_start.py b/integration-tests/tests/test_package_start.py new file mode 100644 index 0000000000..581ccf08e9 --- /dev/null +++ b/integration-tests/tests/test_package_start.py @@ -0,0 +1,65 @@ +"""Integration tests verifying that the CLP package can be started and stopped.""" + +import logging + +import pytest + +from tests.utils.clp_mode_utils import CLP_MODE_CONFIGS +from tests.utils.config import PackageInstance +from tests.utils.package_utils import ( + is_package_running, + is_running_mode_correct, +) + +TEST_MODES = CLP_MODE_CONFIGS.keys() + +logger = logging.getLogger(__name__) + + +@pytest.mark.package +@pytest.mark.parametrize("clp_config", TEST_MODES, indirect=True) +def test_clp_package(clp_package: PackageInstance) -> None: + """ + Validate that all of the components of the CLP package start up successfully for the selected + mode of operation. + """ + # Spin up the package by getting the PackageInstance fixture. + package_instance = clp_package + mode_name = package_instance.package_config.mode_name + instance_id = package_instance.clp_instance_id + + # Ensure that all package components are running. + logger.info( + "Checking if all components of %s package with instance ID '%s' are running properly...", + mode_name, + instance_id, + ) + + running, fail_msg = is_package_running(package_instance) + if not running: + assert fail_msg is not None + pytest.fail(fail_msg) + + logger.info( + "All components of the %s package with instance ID '%s' are running properly.", + mode_name, + instance_id, + ) + + # Ensure that the package is running in the correct mode. + logger.info( + "Checking that the %s package with instance ID '%s' is running in the correct mode...", + mode_name, + instance_id, + ) + + correct, fail_msg = is_running_mode_correct(package_instance) + if not correct: + assert fail_msg is not None + pytest.fail(fail_msg) + + logger.info( + "The %s package with instance ID '%s' is running in the correct mode.", + mode_name, + instance_id, + ) diff --git a/integration-tests/tests/utils/clp_mode_utils.py b/integration-tests/tests/utils/clp_mode_utils.py new file mode 100644 index 0000000000..992eec8c36 --- /dev/null +++ b/integration-tests/tests/utils/clp_mode_utils.py @@ -0,0 +1,97 @@ +"""Provides utilities related to the user-level configurations of CLP's operating modes.""" + +from collections.abc import Callable +from typing import Any + +from clp_py_utils.clp_config import ( + CLPConfig, + COMPRESSION_SCHEDULER_COMPONENT_NAME, + COMPRESSION_WORKER_COMPONENT_NAME, + DB_COMPONENT_NAME, + GARBAGE_COLLECTOR_COMPONENT_NAME, + Package, + QUERY_SCHEDULER_COMPONENT_NAME, + QUERY_WORKER_COMPONENT_NAME, + QueryEngine, + QUEUE_COMPONENT_NAME, + REDIS_COMPONENT_NAME, + REDUCER_COMPONENT_NAME, + RESULTS_CACHE_COMPONENT_NAME, + StorageEngine, + WEBUI_COMPONENT_NAME, +) + + +def _to_container_basename(name: str) -> str: + return name.replace("_", "-") + + +CLP_MODE_CONFIGS: dict[str, tuple[Callable[[], CLPConfig], list[str]]] = { + "clp-text": ( + lambda: CLPConfig( + package=Package( + storage_engine=StorageEngine.CLP, + query_engine=QueryEngine.CLP, + ), + ), + [ + _to_container_basename(DB_COMPONENT_NAME), + _to_container_basename(QUEUE_COMPONENT_NAME), + _to_container_basename(REDIS_COMPONENT_NAME), + _to_container_basename(REDUCER_COMPONENT_NAME), + _to_container_basename(RESULTS_CACHE_COMPONENT_NAME), + _to_container_basename(COMPRESSION_SCHEDULER_COMPONENT_NAME), + _to_container_basename(QUERY_SCHEDULER_COMPONENT_NAME), + _to_container_basename(COMPRESSION_WORKER_COMPONENT_NAME), + _to_container_basename(QUERY_WORKER_COMPONENT_NAME), + _to_container_basename(WEBUI_COMPONENT_NAME), + _to_container_basename(GARBAGE_COLLECTOR_COMPONENT_NAME), + ], + ), + "clp-json": ( + lambda: CLPConfig( + package=Package( + storage_engine=StorageEngine.CLP_S, + query_engine=QueryEngine.CLP_S, + ), + ), + [ + _to_container_basename(DB_COMPONENT_NAME), + _to_container_basename(QUEUE_COMPONENT_NAME), + _to_container_basename(REDIS_COMPONENT_NAME), + _to_container_basename(REDUCER_COMPONENT_NAME), + _to_container_basename(RESULTS_CACHE_COMPONENT_NAME), + _to_container_basename(COMPRESSION_SCHEDULER_COMPONENT_NAME), + _to_container_basename(QUERY_SCHEDULER_COMPONENT_NAME), + _to_container_basename(COMPRESSION_WORKER_COMPONENT_NAME), + _to_container_basename(QUERY_WORKER_COMPONENT_NAME), + _to_container_basename(WEBUI_COMPONENT_NAME), + _to_container_basename(GARBAGE_COLLECTOR_COMPONENT_NAME), + ], + ), +} + + +def compute_mode_signature(config: CLPConfig) -> tuple[Any, ...]: + """Constructs a signature that captures the mode-defining aspects of a CLPConfig object.""" + return ( + config.logs_input.type, + config.package.storage_engine.value, + config.package.query_engine.value, + config.mcp_server is not None, + config.presto is not None, + config.archive_output.storage.type, + config.stream_output.storage.type, + config.aws_config_directory is not None, + config.get_deployment_type(), + ) + + +def get_clp_config_from_mode(mode_name: str) -> CLPConfig: + """Return a CLPConfig object corresponding to the given `mode_name`.""" + try: + config = CLP_MODE_CONFIGS[mode_name][0] + except KeyError as err: + err_msg = f"Unsupported mode: {mode_name}" + raise ValueError(err_msg) from err + return config() diff --git a/integration-tests/tests/utils/config.py b/integration-tests/tests/utils/config.py index 9ef6c101e5..e2bc7552be 100644 --- a/integration-tests/tests/utils/config.py +++ b/integration-tests/tests/utils/config.py @@ -2,12 +2,22 @@ from __future__ import annotations +import re +from collections.abc import Callable from dataclasses import dataclass, field, InitVar from pathlib import Path +import yaml +from clp_py_utils.clp_config import ( + CLP_DEFAULT_LOG_DIRECTORY_PATH, + CLP_SHARED_CONFIG_FILENAME, + CLPConfig, +) + from tests.utils.utils import ( unlink, validate_dir_exists, + validate_file_exists, ) @@ -57,18 +67,35 @@ def clp_s_binary_path(self) -> Path: @dataclass(frozen=True) class PackageConfig: - """The configuration for the clp package subject to testing.""" + """Metadata for the clp package test on this system.""" - #: + #: The directory the package is located in. clp_package_dir: Path + #: Root directory for package tests output. + test_root_dir: Path + + #: Name of the mode of operation represented in this config. + mode_name: str + + #: The CLPConfig object corresponding to this mode of operation. + build_config: Callable[[], CLPConfig] + + #: The list of containerized CLP components that this package needs. + component_list: list[str] + + #: Directory to store any cached package config files. + temp_config_dir: Path = field(init=False, repr=True) + + #: The location of the constructed temporary config file for this package. + temp_config_file_path: Path = field(init=False, repr=True) + def __post_init__(self) -> None: - """Validates that the CLP package directory exists and contains all required directories.""" + """Validates the values specified at init, and initialises attributes.""" + # Validate that the CLP package directory exists and contains all required directories. clp_package_dir = self.clp_package_dir validate_dir_exists(clp_package_dir) - - # Check for required package script directories - required_dirs = ["bin", "etc", "lib", "sbin"] + required_dirs = ["etc", "sbin"] missing_dirs = [d for d in required_dirs if not (clp_package_dir / d).is_dir()] if len(missing_dirs) > 0: err_msg = ( @@ -77,6 +104,107 @@ def __post_init__(self) -> None: ) raise ValueError(err_msg) + # Initialize and create required cache directory for package tests. + object.__setattr__(self, "temp_config_dir", self.test_root_dir / "temp_config_files") + self.test_root_dir.mkdir(parents=True, exist_ok=True) + self.temp_config_dir.mkdir(parents=True, exist_ok=True) + + # Write the temporary config file that the package will use. + clp_config_obj = self.build_config() + temp_config_file_path: Path = self._write_temp_config_file( + clp_config=clp_config_obj, + temp_config_dir=self.temp_config_dir, + mode_name=self.mode_name, + ) + object.__setattr__(self, "temp_config_file_path", temp_config_file_path) + + @property + def start_script_path(self) -> Path: + """:return: The absolute path to the package start script.""" + return self.clp_package_dir / "sbin" / "start-clp.sh" + + @property + def stop_script_path(self) -> Path: + """:return: The absolute path to the package stop script.""" + return self.clp_package_dir / "sbin" / "stop-clp.sh" + + @staticmethod + def _write_temp_config_file( + clp_config: CLPConfig, + temp_config_dir: Path, + mode_name: str, + ) -> Path: + """Writes a temporary config file to `temp_config_dir` for a CLPConfig object.""" + temp_config_dir.mkdir(parents=True, exist_ok=True) + temp_config_filename = f"clp-config-{mode_name}.yml" + temp_config_file_path = temp_config_dir / temp_config_filename + + payload = clp_config.dump_to_primitive_dict() # type: ignore[no-untyped-call] + + tmp_path = temp_config_file_path.with_suffix(temp_config_file_path.suffix + ".tmp") + with tmp_path.open("w", encoding="utf-8") as f: + yaml.safe_dump(payload, f, sort_keys=False) + tmp_path.replace(temp_config_file_path) + + return temp_config_file_path + + +@dataclass(frozen=True) +class PackageInstance: + """Metadata for a run of the clp package.""" + + #: + package_config: PackageConfig + + #: The location of the logging directory within the running package. + clp_log_dir: Path = field(init=False, repr=True) + + #: The instance ID of the running package. + clp_instance_id: str = field(init=False, repr=True) + + #: The path to the .clp-config.yml file constructed by the package during spin-up. + shared_config_file_path: Path = field(init=False, repr=True) + + def __post_init__(self) -> None: + """Validates the values specified at init, and initialises attributes.""" + # Set clp_log_dir and validate that it exists. + clp_log_dir = self.package_config.clp_package_dir / "var" / "log" + validate_dir_exists(clp_log_dir) + object.__setattr__(self, "clp_log_dir", clp_log_dir) + + # Set clp_instance_id. + clp_instance_id_file_path = self.clp_log_dir / "instance-id" + validate_file_exists(clp_instance_id_file_path) + clp_instance_id = self._get_clp_instance_id(clp_instance_id_file_path) + object.__setattr__(self, "clp_instance_id", clp_instance_id) + + # Set shared_config_file_path after validating it. + shared_config_file_path = ( + self.package_config.clp_package_dir + / CLP_DEFAULT_LOG_DIRECTORY_PATH + / CLP_SHARED_CONFIG_FILENAME + ) + validate_file_exists(shared_config_file_path) + object.__setattr__(self, "shared_config_file_path", shared_config_file_path) + + @staticmethod + def _get_clp_instance_id(clp_instance_id_file_path: Path) -> str: + """Return the 4-digit hexadecimal CLP instance-id stored at clp_instance_id_file_path.""" + try: + contents = clp_instance_id_file_path.read_text(encoding="utf-8").strip() + except OSError as err: + err_msg = f"Cannot read instance-id file {clp_instance_id_file_path}: {err}" + raise ValueError(err_msg) from err + + if not re.fullmatch(r"[0-9a-fA-F]{4}", contents): + err_msg = ( + f"Invalid instance ID in {clp_instance_id_file_path}: expected a 4-character" + f" hexadecimal string, but read {contents}." + ) + raise ValueError(err_msg) + + return contents + @dataclass(frozen=True) class IntegrationTestConfig: diff --git a/integration-tests/tests/utils/docker_utils.py b/integration-tests/tests/utils/docker_utils.py new file mode 100644 index 0000000000..5c721fb0aa --- /dev/null +++ b/integration-tests/tests/utils/docker_utils.py @@ -0,0 +1,73 @@ +"""Provide utility functions related to the use of Docker during integration tests.""" + +import re +import subprocess + +DOCKER_STATUS_FIELD_VALS = [ + "created", + "restarting", + "running", + "removing", + "paused", + "exited", + "dead", +] + + +def list_prefixed_containers(docker_bin: str, prefix: str) -> list[str]: + """Returns a list of Docker containers whose names begin with `prefix`.""" + ps_proc = subprocess.run( + [ + docker_bin, + "ps", + "-a", + "--format", + "{{.Names}}", + "--filter", + f"name={prefix}", + ], + capture_output=True, + text=True, + check=False, + ) + + if ps_proc.returncode != 0: + err_out = (ps_proc.stderr or ps_proc.stdout or "").strip() + err_msg = f"Error listing containers for prefix {prefix}: {err_out}" + raise RuntimeError(err_msg) + + candidates: list[str] = [] + for line in (ps_proc.stdout or "").splitlines(): + name_candidate = line.strip() + if re.fullmatch(re.escape(prefix) + r"\d+", name_candidate): + candidates.append(name_candidate) + + return candidates + + +def inspect_container_state(docker_bin: str, name: str, desired_state: str) -> bool: + """ + Inspects the state of the container called `name`. Returns `True` if that container carries + `desired_state`, and `False` otherwise. + """ + if desired_state not in DOCKER_STATUS_FIELD_VALS: + err_msg = f"Unsupported desired_state: {desired_state}" + raise ValueError(err_msg) + + inspect_proc = subprocess.run( + [docker_bin, "inspect", "--format", "{{.State.Status}}", name], + capture_output=True, + text=True, + check=False, + ) + + if inspect_proc.returncode != 0: + err_out = (inspect_proc.stderr or inspect_proc.stdout or "").strip() + if "No such object" in err_out: + err_msg = f"Component container that should be present was not found: {name}" + raise FileNotFoundError(err_msg) + err_msg = f"Error inspecting container {name}: {err_out}" + raise RuntimeError(err_msg) + + status = (inspect_proc.stdout or "").strip().lower() + return status == desired_state diff --git a/integration-tests/tests/utils/package_utils.py b/integration-tests/tests/utils/package_utils.py new file mode 100644 index 0000000000..2c7da02c89 --- /dev/null +++ b/integration-tests/tests/utils/package_utils.py @@ -0,0 +1,116 @@ +"""Provides utility functions related to the clp-package used across `integration-tests`.""" + +import shutil +import subprocess + +from clp_py_utils.clp_config import ( + CLPConfig, +) +from pydantic import ValidationError + +from tests.utils.clp_mode_utils import compute_mode_signature, get_clp_config_from_mode +from tests.utils.config import ( + PackageConfig, + PackageInstance, +) +from tests.utils.docker_utils import ( + inspect_container_state, + list_prefixed_containers, +) +from tests.utils.utils import load_yaml_to_dict + + +def start_clp_package(package_config: PackageConfig) -> None: + """Start an instance of the CLP package.""" + start_script_path = package_config.start_script_path + try: + # fmt: off + start_cmd = [ + str(start_script_path), + "--config", + str(package_config.temp_config_file_path) + ] + # fmt: on + subprocess.run(start_cmd, check=True) + except Exception as e: + err_msg = f"Failed to start an instance of the {package_config.mode_name} package." + raise RuntimeError(err_msg) from e + + +def stop_clp_package(instance: PackageInstance) -> None: + """Stop an instance of the CLP package.""" + package_config = instance.package_config + stop_script_path = package_config.stop_script_path + try: + # fmt: off + stop_cmd = [ + stop_script_path + ] + # fmt: on + subprocess.run(stop_cmd, check=True) + except Exception as e: + err_msg = f"Failed to stop an instance of the {package_config.mode_name} package." + raise RuntimeError(err_msg) from e + + +def is_package_running(package_instance: PackageInstance) -> tuple[bool, str | None]: + """ + Checks that the `package_instance` is running properly by examining each of its component + containers. Records which containers are not found or not running. + """ + docker_bin = shutil.which("docker") + if docker_bin is None: + err_msg = "docker not found in PATH" + raise RuntimeError(err_msg) + + instance_id = package_instance.clp_instance_id + problems: list[str] = [] + + required_components = package_instance.package_config.component_list + for component in required_components: + prefix = f"clp-package-{instance_id}-{component}-" + + candidates = list_prefixed_containers(docker_bin, prefix) + if not candidates: + problems.append(f"No component container was found with the prefix '{prefix}'") + continue + + not_running: list[str] = [] + for name in candidates: + if not inspect_container_state(docker_bin, name, "running"): + not_running.append(name) + + if not_running: + details = ", ".join(not_running) + problems.append(f"Component containers not running: {details}") + + if problems: + return False, "; ".join(problems) + + return True, None + + +def is_running_mode_correct(package_instance: PackageInstance) -> tuple[bool, str | None]: + """ + Checks if the mode described in the shared config file of `package_instance` is accurate with + respect to its `mode_name`. Returns `True` if correct, `False` with message on mismatch. + """ + shared_config_dict = load_yaml_to_dict(package_instance.shared_config_file_path) + try: + running_config = CLPConfig.model_validate(shared_config_dict) + except ValidationError as err: + err_msg = f"Shared config failed validation: {err}" + raise ValueError(err_msg) from err + + intended_config = get_clp_config_from_mode(package_instance.package_config.mode_name) + + running_signature = compute_mode_signature(running_config) + intended_signature = compute_mode_signature(intended_config) + + if running_signature != intended_signature: + return ( + False, + "Mode mismatch: running configuration does not match intended configuration.", + ) + + return True, None diff --git a/integration-tests/tests/utils/utils.py b/integration-tests/tests/utils/utils.py index 1dca8ba162..075359e589 100644 --- a/integration-tests/tests/utils/utils.py +++ b/integration-tests/tests/utils/utils.py @@ -5,7 +5,9 @@ import subprocess from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO +from typing import Any, IO + +import yaml def get_env_var(var_name: str) -> str: @@ -51,6 +53,31 @@ def is_json_file_structurally_equal(json_fp1: Path, json_fp2: Path) -> bool: return is_dir_tree_content_equal(Path(temp_file_1.name), Path(temp_file_2.name)) +def load_yaml_to_dict(path: Path) -> dict[str, Any]: + """ + :param path: + :return: The dict constructed from parsing the content of the YAML file at `path`. + :raise: ValueError if there is invalid YAML in target file. + :raise: ValueError if the target file can't be read. + :raise: TypeError if the target file has no top-level mapping. + """ + try: + with path.open("r", encoding="utf-8") as file: + target_dict = yaml.safe_load(file) + except yaml.YAMLError as err: + err_msg = f"Invalid YAML in target file {path}: {err}" + raise ValueError(err_msg) from err + except OSError as err: + err_msg = f"Cannot read target file {path}: {err}" + raise ValueError(err_msg) from err + + if not isinstance(target_dict, dict): + err_msg = f"Target file {path} must have a top-level mapping." + raise TypeError(err_msg) + + return target_dict + + def unlink(rm_path: Path, force: bool = True) -> None: """ Remove a file or directory at `path`. @@ -85,6 +112,16 @@ def validate_dir_exists(dir_path: Path) -> None: raise ValueError(err_msg) +def validate_file_exists(file_path: Path) -> None: + """ + :param file_path: + :raise: ValueError if the path does not exist or is not a file. + """ + if not file_path.is_file(): + err_msg = f"Path does not exist or is not a file: {file_path}" + raise ValueError(err_msg) + + def _sort_json_keys_and_rows(json_fp: Path) -> IO[str]: """ Normalize a JSON file to a stable, deterministically ordered form for comparison. diff --git a/integration-tests/uv.lock b/integration-tests/uv.lock index 4145ce1d7c..c6cea155dc 100644 --- a/integration-tests/uv.lock +++ b/integration-tests/uv.lock @@ -882,7 +882,9 @@ dev = [ { name = "mypy" }, { name = "pytest" }, { name = "pytest-env" }, + { name = "pyyaml" }, { name = "ruff" }, + { name = "types-pyyaml" }, ] [package.metadata] @@ -898,7 +900,9 @@ dev = [ { name = "mypy", specifier = ">=1.16.0" }, { name = "pytest", specifier = ">=8.4.1" }, { name = "pytest-env", specifier = ">=1.1.5" }, + { name = "pyyaml", specifier = ">=6.0" }, { name = "ruff", specifier = ">=0.11.12" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20240808" }, ] [[package]] @@ -2475,6 +2479,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250915" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/69/3c51b36d04da19b92f9e815be12753125bd8bc247ba0470a982e6979e71c/types_pyyaml-6.0.12.20250915.tar.gz", hash = "sha256:0f8b54a528c303f0e6f7165687dd33fafa81c807fcac23f632b63aa624ced1d3", size = 17522, upload-time = "2025-09-15T03:01:00.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/e0/1eed384f02555dde685fff1a1ac805c1c7dcb6dd019c916fe659b1c1f9ec/types_pyyaml-6.0.12.20250915-py3-none-any.whl", hash = "sha256:e7d4d9e064e89a3b3cae120b4990cd370874d2bf12fa5f46c97018dd5d3c9ab6", size = 20338, upload-time = "2025-09-15T03:00:59.218Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" diff --git a/taskfiles/tests/integration.yaml b/taskfiles/tests/integration.yaml index 2b31492002..5af99807d7 100644 --- a/taskfiles/tests/integration.yaml +++ b/taskfiles/tests/integration.yaml @@ -29,3 +29,13 @@ tasks: clp-py-project-imports: dir: "{{.G_INTEGRATION_TESTS_DIR}}" cmd: "uv run python -m pytest tests/test_clp_native_py_project_imports.py" + + package: + deps: + - task: "::package" + dir: "{{.G_INTEGRATION_TESTS_DIR}}" + env: + CLP_BUILD_DIR: "{{.G_BUILD_DIR}}" + CLP_CORE_BINS_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}" + CLP_PACKAGE_DIR: "{{.G_PACKAGE_BUILD_DIR}}" + cmd: "uv run python -m pytest -m package"