diff --git a/.github/workflows/build_and_test_on_every_pr.yml b/.github/workflows/build_and_test_on_every_pr.yml index c1c32865e6..a1cfd2c019 100644 --- a/.github/workflows/build_and_test_on_every_pr.yml +++ b/.github/workflows/build_and_test_on_every_pr.yml @@ -51,4 +51,6 @@ jobs: while read -r target; do bazel run --config bl-x86_64-linux "$target" done < ci/showcase_targets_run.txt - \ No newline at end of file + - name: Feature Integration Tests + run: | + bazel run --config bl-x86_64-linux //feature_integration_tests/python_test_cases:fit \ No newline at end of file diff --git a/.gitignore b/.gitignore index a983076b26..db54c5cd2d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,11 @@ _logs .ruff_cache target/ -rust-project.json \ No newline at end of file +rust-project.json + +# Python +.venv +__pycache__/ +.pytest_cache/ +/.coverage +**/*.egg-info/* \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 853462a098..a3e396b541 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -34,6 +34,15 @@ python.toolchain( ) use_repo(python) +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True) +pip.parse( + hub_name = "pip_score_venv_test", + python_version = PYTHON_VERSION, + requirements_lock = "//feature_integration_tests/python_test_cases:requirements.txt.lock", +) + +use_repo(pip, "pip_score_venv_test") + # Special imports for certain modules # communication module dependencies diff --git a/feature_integration_tests/README.md b/feature_integration_tests/README.md new file mode 100644 index 0000000000..902e6d11fb --- /dev/null +++ b/feature_integration_tests/README.md @@ -0,0 +1,45 @@ +# Feature Integration Tests + +This directory contains Feature Integration Tests for the S-CORE project. It includes both Python test cases and Rust test scenarios to validate features work together. + +## Structure + +- `python_test_cases/` — Python-based integration test cases + - `conftest.py` — Pytest configuration and fixtures + - `fit_scenario.py` — Base scenario class + - `requirements.txt` — Python dependencies + - `BUILD` — Bazel build and test definitions + - `tests/` — Test cases (e.g., orchestration with persistency) +- `rust_test_scenarios/` — Rust-based integration test scenarios + - `src/` — Rust source code for test scenarios + - `BUILD` — Bazel build definitions + +## Running Tests + +### Python Test Cases + +Python tests are managed with Bazel and Pytest. To run the main test target: + +```sh +bazel test //feature_integration_tests/python_test_cases:fit +``` + +### Rust Test Scenarios + +Rust test scenarios are defined in `rust_test_scenarios/src/scenarios`. Build and run them using Bazel: + +```sh +bazel build //feature_integration_tests/rust_test_scenarios +``` + +```sh +bazel run //feature_integration_tests/rust_test_scenarios -- --list-scenarios +``` + +## Updating Python Requirements + +To update Python dependencies: + +```sh +bazel run //feature_integration_tests/python_test_cases:requirements.update +``` diff --git a/feature_integration_tests/python_test_cases/BUILD b/feature_integration_tests/python_test_cases/BUILD new file mode 100644 index 0000000000..3aa8031e9b --- /dev/null +++ b/feature_integration_tests/python_test_cases/BUILD @@ -0,0 +1,45 @@ +load("@pip_score_venv_test//:requirements.bzl", "all_requirements") +load("@rules_python//python:pip.bzl", "compile_pip_requirements") +load("@score_tooling//python_basics:defs.bzl", "score_py_pytest", "score_virtualenv") + +# In order to update the requirements, change the `requirements.txt` file and run: +# `bazel run //feature_integration_tests/python_test_cases:requirements.update`. +# This will update the `requirements.txt.lock` file. +# To upgrade all dependencies to their latest versions, run: +# `bazel run //feature_integration_tests/python_test_cases:requirements.update -- --upgrade`. +compile_pip_requirements( + name = "requirements", + srcs = [ + "requirements.txt", + "@score_tooling//python_basics:requirements.txt", + ], + requirements_txt = "requirements.txt.lock", + tags = [ + "manual", + ], +) + +score_virtualenv( + name = "python_tc_venv", + reqs = all_requirements, + venv_name = ".python_tc_venv", +) + +# Tests targets +score_py_pytest( + name = "fit", + srcs = glob(["tests/**/*.py"]) + ["conftest.py", "fit_scenario.py"], + args = [ + "--traces=all", + "--rust-target-path=$(rootpath //feature_integration_tests/rust_test_scenarios)", + ], + data = [ + ":python_tc_venv", + "//feature_integration_tests/rust_test_scenarios", + ], + env = { + "RUST_BACKTRACE": "1", + }, + pytest_ini = ":pytest.ini", + deps = all_requirements, +) diff --git a/feature_integration_tests/python_test_cases/conftest.py b/feature_integration_tests/python_test_cases/conftest.py new file mode 100644 index 0000000000..001b14a2b3 --- /dev/null +++ b/feature_integration_tests/python_test_cases/conftest.py @@ -0,0 +1,66 @@ +from pathlib import Path + +import pytest +from testing_utils import BazelTools + +FAILED_CONFIGS = [] + + +# Cmdline options +def pytest_addoption(parser): + parser.addoption( + "--traces", + choices=["none", "target", "all"], + default="none", + help="Verbosity of traces in output and HTML report. " + '"none" - show no traces, ' + '"target" - show traces generated by test code, ' + '"all" - show all traces. ', + ) + + parser.addoption( + "--rust-target-name", + type=str, + default="//feature_integration_tests/rust_test_scenarios:rust_test_scenarios", + help="Rust test scenario executable target.", + ) + parser.addoption( + "--rust-target-path", + type=Path, + help="Rust test scenario executable target.", + ) + parser.addoption( + "--build-scenarios", + action="store_true", + help="Build test scenarios executables.", + ) + parser.addoption( + "--build-scenarios-timeout", + type=float, + default=180.0, + help="Build command timeout in seconds. Default: %(default)s", + ) + parser.addoption( + "--default-execution-timeout", + type=float, + default=5.0, + help="Default execution timeout in seconds. Default: %(default)s", + ) + + +# Hooks +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session): + try: + # Build scenarios. + if session.config.getoption("--build-scenarios"): + build_timeout = session.config.getoption("--build-scenarios-timeout") + + # Build Rust test scenarios. + print("Building Rust test scenarios executable...") + cargo_tools = BazelTools(option_prefix="rust", build_timeout=build_timeout) + rust_target_name = session.config.getoption("--rust-target-name") + cargo_tools.build(rust_target_name) + + except Exception as e: + pytest.exit(str(e), returncode=1) diff --git a/feature_integration_tests/python_test_cases/fit_scenario.py b/feature_integration_tests/python_test_cases/fit_scenario.py new file mode 100644 index 0000000000..f76406d614 --- /dev/null +++ b/feature_integration_tests/python_test_cases/fit_scenario.py @@ -0,0 +1,135 @@ +import shutil +from pathlib import Path +from typing import Generator + +import pytest +from testing_utils import ( + BazelTools, + BuildTools, + LogContainer, + Scenario, + ScenarioResult, +) + + +def temp_dir_common( + tmp_path_factory: pytest.TempPathFactory, base_name: str, *args: str +) -> Generator[Path, None, None]: + """ + Create temporary directory and remove it after test. + Common implementation to be reused by fixtures. + + Returns generator providing numbered path to temporary directory. + E.g., '/--/'. + + Parameters + ---------- + tmp_path_factory : pytest.TempPathFactory + Factory for temporary directories. + base_name : str + Base directory name. + 'self.__class__.__name__' use is recommended. + *args : Any + Other parameters to be included in directory name. + """ + parts = [base_name, *args] + dir_name = "-".join(parts) + dir_path = tmp_path_factory.mktemp(dir_name, numbered=True) + yield dir_path + shutil.rmtree(dir_path) + + +class FitScenario(Scenario): + """ + CIT test scenario definition. + """ + + @pytest.fixture(scope="class") + def build_tools(self) -> BuildTools: + return BazelTools(option_prefix="rust") + + def expect_command_failure(self, *args, **kwargs) -> bool: + """ + Expect command failure (e.g., non-zero return code or hang). + """ + return False + + @pytest.fixture(scope="class") + def results( + self, + command: list[str], + execution_timeout: float, + *args, + **kwargs, + ) -> ScenarioResult: + result = self._run_command(command, execution_timeout, args, kwargs) + success = result.return_code == 0 and not result.hang + if self.expect_command_failure() and success: + raise RuntimeError(f"Command execution succeeded unexpectedly: {result=}") + if not self.expect_command_failure() and not success: + raise RuntimeError(f"Command execution failed unexpectedly: {result=}") + return result + + @pytest.fixture(scope="class") + def logs_target(self, target_path: Path, logs: LogContainer) -> LogContainer: + """ + Logs with messages generated strictly by the tested code. + + Parameters + ---------- + target_path : Path + Path to test scenario executable. + logs : LogContainer + Unfiltered logs. + """ + return logs.get_logs(field="target", pattern=f"{target_path.name}.*") + + @pytest.fixture(scope="class") + def logs_info_level(self, logs_target: LogContainer) -> LogContainer: + """ + Logs with messages with INFO level. + + Parameters + ---------- + logs_target : LogContainer + Logs with messages generated strictly by the tested code. + """ + return logs_target.get_logs(field="level", value="INFO") + + @pytest.fixture(autouse=True) + def print_to_report( + self, + request: pytest.FixtureRequest, + logs: LogContainer, + logs_target: LogContainer, + ) -> None: + """ + Print traces to stdout. + + Allowed "--traces" values: + - "none" - show no traces. + - "target" - show traces generated by test code. + - "all" - show all traces. + + Parameters + ---------- + request : FixtureRequest + Test request built-in fixture. + logs : LogContainer + Test scenario execution logs. + logs_target : LogContainer + Logs with messages generated strictly by the tested code. + """ + traces_param = request.config.getoption("--traces") + match traces_param: + case "all": + traces = logs + case "target": + traces = logs_target + case "none": + traces = LogContainer() + case _: + raise RuntimeError(f'Invalid "--traces" value: {traces_param}') + + for trace in traces: + print(trace) diff --git a/feature_integration_tests/python_test_cases/pytest.ini b/feature_integration_tests/python_test_cases/pytest.ini new file mode 100644 index 0000000000..d4b25fd2f8 --- /dev/null +++ b/feature_integration_tests/python_test_cases/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +addopts = -v +testpaths = tests diff --git a/feature_integration_tests/python_test_cases/requirements.txt b/feature_integration_tests/python_test_cases/requirements.txt new file mode 100644 index 0000000000..8bbb929d2d --- /dev/null +++ b/feature_integration_tests/python_test_cases/requirements.txt @@ -0,0 +1 @@ +testing-utils @ git+https://github.com/eclipse-score/testing_tools.git@v0.3.0 diff --git a/feature_integration_tests/python_test_cases/requirements.txt.lock b/feature_integration_tests/python_test_cases/requirements.txt.lock new file mode 100644 index 0000000000..3eb053f6e2 --- /dev/null +++ b/feature_integration_tests/python_test_cases/requirements.txt.lock @@ -0,0 +1,133 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# bazel run //feature_integration_tests/python_test_cases:requirements.update +# +basedpyright==1.29.2 \ + --hash=sha256:12c49186003b9f69a028615da883ef97035ea2119a9e3f93a00091b3a27088a6 \ + --hash=sha256:f389e2997de33d038c5065fd85bff351fbdc62fa6d6371c7b947fc3bce8d437d + # via -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt +iniconfig==2.1.0 \ + --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \ + --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + # via + # -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt + # pytest +jinja2==3.1.6 \ + --hash=sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d \ + --hash=sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67 + # via pytest-html +markupsafe==3.0.2 \ + --hash=sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4 \ + --hash=sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30 \ + --hash=sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0 \ + --hash=sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9 \ + --hash=sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396 \ + --hash=sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13 \ + --hash=sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028 \ + --hash=sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca \ + --hash=sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557 \ + --hash=sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832 \ + --hash=sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0 \ + --hash=sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b \ + --hash=sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579 \ + --hash=sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a \ + --hash=sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c \ + --hash=sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff \ + --hash=sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c \ + --hash=sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22 \ + --hash=sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094 \ + --hash=sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb \ + --hash=sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e \ + --hash=sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5 \ + --hash=sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a \ + --hash=sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d \ + --hash=sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a \ + --hash=sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b \ + --hash=sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8 \ + --hash=sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225 \ + --hash=sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c \ + --hash=sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144 \ + --hash=sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f \ + --hash=sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87 \ + --hash=sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d \ + --hash=sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93 \ + --hash=sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf \ + --hash=sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158 \ + --hash=sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84 \ + --hash=sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb \ + --hash=sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48 \ + --hash=sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171 \ + --hash=sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c \ + --hash=sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6 \ + --hash=sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd \ + --hash=sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d \ + --hash=sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1 \ + --hash=sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d \ + --hash=sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca \ + --hash=sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a \ + --hash=sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29 \ + --hash=sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe \ + --hash=sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798 \ + --hash=sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c \ + --hash=sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8 \ + --hash=sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f \ + --hash=sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f \ + --hash=sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a \ + --hash=sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178 \ + --hash=sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0 \ + --hash=sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79 \ + --hash=sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430 \ + --hash=sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50 + # via jinja2 +nodejs-wheel-binaries==22.16.0 \ + --hash=sha256:2728972d336d436d39ee45988978d8b5d963509e06f063e80fe41b203ee80b28 \ + --hash=sha256:2fffb4bf1066fb5f660da20819d754f1b424bca1b234ba0f4fa901c52e3975fb \ + --hash=sha256:447ad796850eb52ca20356ad39b2d296ed8fef3f214921f84a1ccdad49f2eba1 \ + --hash=sha256:4ae3cf22138891cb44c3ee952862a257ce082b098b29024d7175684a9a77b0c0 \ + --hash=sha256:71f2de4dc0b64ae43e146897ce811f80ac4f9acfbae6ccf814226282bf4ef174 \ + --hash=sha256:7f526ca6a132b0caf633566a2a78c6985fe92857e7bfdb37380f76205a10b808 \ + --hash=sha256:986b715a96ed703f8ce0c15712f76fc42895cf09067d72b6ef29e8b334eccf64 \ + --hash=sha256:d695832f026df3a0cf9a089d222225939de9d1b67f8f0a353b79f015aabbe7e2 \ + --hash=sha256:dbfccbcd558d2f142ccf66d8c3a098022bf4436db9525b5b8d32169ce185d99e + # via + # -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt + # basedpyright +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via + # -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt + # pytest +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via + # -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt + # pytest +pytest==8.3.5 \ + --hash=sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820 \ + --hash=sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845 + # via + # -r /home/pko/.cache/bazel/_bazel_pko/2af2ff4db91047a487505aa8f91f8f54/external/score_tooling+/python_basics/requirements.txt + # pytest-html + # pytest-metadata + # pytest-repeat + # testing-utils +pytest-html==4.1.1 \ + --hash=sha256:70a01e8ae5800f4a074b56a4cb1025c8f4f9b038bba5fe31e3c98eb996686f07 \ + --hash=sha256:c8152cea03bd4e9bee6d525573b67bbc6622967b72b9628dda0ea3e2a0b5dd71 + # via testing-utils +pytest-metadata==3.1.1 \ + --hash=sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b \ + --hash=sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8 + # via pytest-html +pytest-repeat==0.9.4 \ + --hash=sha256:c1738b4e412a6f3b3b9e0b8b29fcd7a423e50f87381ad9307ef6f5a8601139f3 \ + --hash=sha256:d92ac14dfaa6ffcfe6917e5d16f0c9bc82380c135b03c2a5f412d2637f224485 + # via testing-utils +# WARNING: pip install will require the following package to be hashed. +# Consider using a hashable URL like https://github.com/jazzband/pip-tools/archive/SOMECOMMIT.zip +testing-utils @ git+https://github.com/eclipse-score/testing_tools.git@v0.3.0 + # via -r feature_integration_tests/python_test_cases/requirements.txt diff --git a/feature_integration_tests/python_test_cases/tests/basic/test_orchestartion_with_persistency.py b/feature_integration_tests/python_test_cases/tests/basic/test_orchestartion_with_persistency.py new file mode 100644 index 0000000000..201851da83 --- /dev/null +++ b/feature_integration_tests/python_test_cases/tests/basic/test_orchestartion_with_persistency.py @@ -0,0 +1,52 @@ +import json +from pathlib import Path +from typing import Any, Generator + +import pytest +from fit_scenario import FitScenario, temp_dir_common +from testing_utils import LogContainer + + +class TestOrchWithPersistency(FitScenario): + """ + Tests orchestration with persistency scenario. + Scenario uses Orchestration and Kyron to run program `run_count` times. + Each run increments counter stored by KVS in `tmp_dir`. + After all runs, test verifies that counter value equals `run_count`. + """ + + @pytest.fixture(scope="class") + def scenario_name(self) -> str: + return "basic.orchestration_with_persistency" + + @pytest.fixture(scope="class", params=[1, 5]) + def run_count(self, request) -> int: + return request.param + + @pytest.fixture(scope="class") + def temp_dir( + self, + tmp_path_factory: pytest.TempPathFactory, + run_count: int, # run_count is required to ensure proper order of fixture calls + ) -> Generator[Path, None, None]: + yield from temp_dir_common(tmp_path_factory, self.__class__.__name__) + + @pytest.fixture(scope="class") + def test_config(self, run_count: int, temp_dir: Path) -> dict[str, Any]: + return { + "runtime": {"task_queue_size": 2, "workers": 4}, + "test": {"run_count": run_count, "kvs_path": str(temp_dir)}, + } + + def test_kvs_logged_execution(self, run_count: int, logs_info_level: LogContainer): + """Verify that all runs have been logged.""" + logs = logs_info_level.get_logs(field="run_cycle_number") + logged_cycles = [log.run_cycle_number for log in logs] + expected_cycles = list(range(1, run_count + 1)) + assert logged_cycles == expected_cycles + + def test_kvs_write_results(self, temp_dir: Path, run_count: int): + """Verify that KVS file contains correct final run count.""" + kvs_file = temp_dir / "kvs_1_0.json" + data = json.loads(kvs_file.read_text()) + assert data["v"]["run_cycle_number"]["v"] == run_count diff --git a/feature_integration_tests/rust_test_scenarios/BUILD b/feature_integration_tests/rust_test_scenarios/BUILD index 78d0b3b53c..d6c2e0db3c 100644 --- a/feature_integration_tests/rust_test_scenarios/BUILD +++ b/feature_integration_tests/rust_test_scenarios/BUILD @@ -16,6 +16,7 @@ load("@rules_rust//rust:defs.bzl", "rust_binary") rust_binary( name = "rust_test_scenarios", srcs = glob(["src/**/*.rs"]), + visibility = ["//feature_integration_tests/python_test_cases:__pkg__"], tags = [ "manual", ], @@ -26,7 +27,8 @@ rust_binary( "@score_persistency//src/rust/rust_kvs:rust_kvs", "@score_test_scenarios//test_scenarios_rust:test_scenarios_rust", "@score_crates//:tracing", + "@score_crates//:tracing_subscriber", "@score_crates//:serde", "@score_crates//:serde_json", - ] + ], ) \ No newline at end of file diff --git a/feature_integration_tests/rust_test_scenarios/src/internals/runtime_helper.rs b/feature_integration_tests/rust_test_scenarios/src/internals/runtime_helper.rs index 341d22be3c..3713d23566 100644 --- a/feature_integration_tests/rust_test_scenarios/src/internals/runtime_helper.rs +++ b/feature_integration_tests/rust_test_scenarios/src/internals/runtime_helper.rs @@ -97,12 +97,16 @@ impl Runtime { serde_json::from_value(v["runtime"].clone()).map_err(|e| e.to_string()) } + #[allow(dead_code)] pub fn exec_engines(&self) -> &Vec { &self.exec_engines } pub fn build(&self) -> kyron::runtime::Runtime { - debug!("Creating kyron::Runtime with {} execution engines", self.exec_engines.len()); + debug!( + "Creating kyron::Runtime with {} execution engines", + self.exec_engines.len() + ); let mut async_rt_builder = kyron::runtime::RuntimeBuilder::new(); for exec_engine in self.exec_engines.as_slice() { @@ -134,23 +138,34 @@ impl Runtime { for dedicated_worker in dedicated_workers { // Create thread parameters object. let mut async_rt_thread_params = AsyncRtThreadParameters::default(); - if let Some(thread_priority) = dedicated_worker.thread_parameters.thread_priority { + if let Some(thread_priority) = + dedicated_worker.thread_parameters.thread_priority + { async_rt_thread_params = async_rt_thread_params.priority(thread_priority); } - if let Some(thread_affinity) = &dedicated_worker.thread_parameters.thread_affinity { + if let Some(thread_affinity) = + &dedicated_worker.thread_parameters.thread_affinity + { async_rt_thread_params = async_rt_thread_params.affinity(thread_affinity); } - if let Some(thread_stack_size) = dedicated_worker.thread_parameters.thread_stack_size { - async_rt_thread_params = async_rt_thread_params.stack_size(thread_stack_size); + if let Some(thread_stack_size) = + dedicated_worker.thread_parameters.thread_stack_size + { + async_rt_thread_params = + async_rt_thread_params.stack_size(thread_stack_size); } - if let Some(thread_scheduler) = dedicated_worker.thread_parameters.thread_scheduler { - async_rt_thread_params = async_rt_thread_params.scheduler_type(thread_scheduler); + if let Some(thread_scheduler) = + dedicated_worker.thread_parameters.thread_scheduler + { + async_rt_thread_params = + async_rt_thread_params.scheduler_type(thread_scheduler); } // Create `UniqueWorkerId`. let unique_worker_id = UniqueWorkerId::from(&dedicated_worker.id); - exec_engine_builder = exec_engine_builder.with_dedicated_worker(unique_worker_id, async_rt_thread_params); + exec_engine_builder = exec_engine_builder + .with_dedicated_worker(unique_worker_id, async_rt_thread_params); } } @@ -158,6 +173,8 @@ impl Runtime { async_rt_builder = builder; } - async_rt_builder.build().expect("Failed to build async runtime") + async_rt_builder + .build() + .expect("Failed to build async runtime") } } diff --git a/feature_integration_tests/rust_test_scenarios/src/main.rs b/feature_integration_tests/rust_test_scenarios/src/main.rs index 4220d66f91..1b6fd78e5c 100644 --- a/feature_integration_tests/rust_test_scenarios/src/main.rs +++ b/feature_integration_tests/rust_test_scenarios/src/main.rs @@ -12,12 +12,38 @@ // mod internals; -mod tests; +mod scenarios; use test_scenarios_rust::cli::run_cli_app; use test_scenarios_rust::test_context::TestContext; -use crate::tests::root_scenario_group; +use crate::scenarios::root_scenario_group; +use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::Level; +use tracing_subscriber::fmt::time::FormatTime; +use tracing_subscriber::FmtSubscriber; +struct NumericUnixTime; + +impl FormatTime for NumericUnixTime { + fn format_time(&self, w: &mut tracing_subscriber::fmt::format::Writer<'_>) -> std::fmt::Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + write!(w, "{}", now.as_secs()) + } +} + +fn init_tracing_subscriber() { + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::TRACE) + .with_thread_ids(true) + .with_timer(NumericUnixTime) + .json() + .finish(); + + tracing::subscriber::set_global_default(subscriber) + .expect("Setting default subscriber failed!"); +} fn main() -> Result<(), String> { let raw_arguments: Vec = std::env::args().collect(); @@ -26,6 +52,7 @@ fn main() -> Result<(), String> { let root_group = root_scenario_group(); // Run. + init_tracing_subscriber(); let test_context = TestContext::new(root_group); run_cli_app(&raw_arguments, &test_context) } diff --git a/feature_integration_tests/rust_test_scenarios/src/tests/basic/mod.rs b/feature_integration_tests/rust_test_scenarios/src/scenarios/basic/mod.rs similarity index 66% rename from feature_integration_tests/rust_test_scenarios/src/tests/basic/mod.rs rename to feature_integration_tests/rust_test_scenarios/src/scenarios/basic/mod.rs index 3c1634b160..62667b93a6 100644 --- a/feature_integration_tests/rust_test_scenarios/src/tests/basic/mod.rs +++ b/feature_integration_tests/rust_test_scenarios/src/scenarios/basic/mod.rs @@ -1,4 +1,3 @@ -// // Copyright (c) 2025 Contributors to the Eclipse Foundation // // See the NOTICE file(s) distributed with this work for additional @@ -10,27 +9,16 @@ // // SPDX-License-Identifier: Apache-2.0 // -use orchestration::prelude::*; use test_scenarios_rust::scenario::{ScenarioGroup, ScenarioGroupImpl}; -use tracing::info; - -mod only_shutdown_sequence; -fn simple_checkpoint(id: &str) { - info!(id = id); -} - -async fn basic_task() -> InvokeResult { - simple_checkpoint("basic_task"); - Ok(()) -} +mod orchestration_with_persistency; pub fn basic_scenario_group() -> Box { Box::new(ScenarioGroupImpl::new( "basic", - vec![ - Box::new(only_shutdown_sequence::OnlyShutdownSequence), - ], + vec![Box::new( + orchestration_with_persistency::OrchestrationWithPersistency, + )], vec![], )) } diff --git a/feature_integration_tests/rust_test_scenarios/src/scenarios/basic/orchestration_with_persistency.rs b/feature_integration_tests/rust_test_scenarios/src/scenarios/basic/orchestration_with_persistency.rs new file mode 100644 index 0000000000..a9a7641a77 --- /dev/null +++ b/feature_integration_tests/rust_test_scenarios/src/scenarios/basic/orchestration_with_persistency.rs @@ -0,0 +1,149 @@ +// +// Copyright (c) 2025 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// +use crate::internals::runtime_helper::Runtime; +use kyron_foundation::containers::Vector; +use kyron_foundation::prelude::CommonErrors; +use orchestration::prelude::*; +use orchestration::{ + api::{design::Design, Orchestration}, + common::DesignConfig, +}; + +use rust_kvs::kvs_api::KvsApi; +use rust_kvs::Kvs; +use rust_kvs::KvsBuilder; +use serde_json::Value; + +use test_scenarios_rust::scenario::Scenario; +use tracing::info; + +use serde::Deserialize; + +#[derive(Deserialize, Debug)] +struct TestInput { + run_count: usize, + kvs_path: String, +} + +impl TestInput { + pub fn new(input: &str) -> Self { + let v: Value = serde_json::from_str(input).expect("Failed to parse input string"); + serde_json::from_value(v["test"].clone()).expect("Failed to parse \"test\" field") + } +} + +use rust_kvs::prelude::{InstanceId, KvsDefaults, KvsLoad}; +use std::path::PathBuf; +pub struct KvsParameters { + pub instance_id: InstanceId, + pub defaults: Option, + pub kvs_load: Option, + pub dir: Option, + pub snapshot_max_count: Option, +} + +macro_rules! persistency_task { + ($path:expr) => { + move || kvs_save_cycle_number($path.clone()) + }; +} + +async fn kvs_save_cycle_number(path: String) -> Result<(), UserErrValue> { + let params = KvsParameters { + instance_id: InstanceId(1), + defaults: Some(KvsDefaults::Optional), + kvs_load: Some(KvsLoad::Optional), + dir: Some(PathBuf::from(path)), + snapshot_max_count: Some(3), + }; + + // Set builder parameters. + let mut builder = KvsBuilder::new(params.instance_id); + if let Some(flag) = params.defaults { + builder = builder.defaults(flag); + } + if let Some(flag) = params.kvs_load { + builder = builder.kvs_load(flag); + } + if let Some(dir) = params.dir { + builder = builder.dir(dir.to_string_lossy().to_string()); + } + if let Some(max_count) = params.snapshot_max_count { + builder = builder.snapshot_max_count(max_count); + } + + // Create KVS. + let kvs: Kvs = builder.build().expect("Failed to build KVS instance"); + + // Simple set/get. + let key = "run_cycle_number"; + let last_cycle_number: u32 = kvs.get_value_as::(key).unwrap_or_else(|_| 0_u32); + + kvs.set_value(key, last_cycle_number + 1) + .expect("Failed to set value"); + let value_read = kvs.get_value_as::(key).expect("Failed to read value"); + + kvs.flush().expect("Failed to flush KVS"); + + info!(run_cycle_number = value_read); + + Ok(()) +} + +fn single_sequence_design(kvs_path: String) -> Result { + let mut design = Design::new("SingleSequence".into(), DesignConfig::default()); + let kvs_cycle_tag = + design.register_invoke_async("KVS save cycle".into(), persistency_task!(kvs_path))?; + + // Create a program with actions + design.add_program(file!(), move |_design_instance, builder| { + builder.with_run_action( + SequenceBuilder::new() + .with_step(Invoke::from_tag(&kvs_cycle_tag, _design_instance.config())) + .build(), + ); + + Ok(()) + }); + + Ok(design) +} + +pub struct OrchestrationWithPersistency; + +impl Scenario for OrchestrationWithPersistency { + fn name(&self) -> &str { + "orchestration_with_persistency" + } + + fn run(&self, input: &str) -> Result<(), String> { + let logic = TestInput::new(input); + let mut rt = Runtime::from_json(input)?.build(); + + let orch = Orchestration::new() + .add_design(single_sequence_design(logic.kvs_path).expect("Failed to create design")) + .design_done(); + + let mut program_manager = orch + .into_program_manager() + .expect("Failed to create programs"); + let mut programs = program_manager.get_programs(); + + rt.block_on(async move { + let mut program = programs.pop().expect("Failed to pop program"); + let _ = program.run_n(logic.run_count).await; + }); + + Ok(()) + } +} diff --git a/feature_integration_tests/rust_test_scenarios/src/tests/mod.rs b/feature_integration_tests/rust_test_scenarios/src/scenarios/mod.rs similarity index 100% rename from feature_integration_tests/rust_test_scenarios/src/tests/mod.rs rename to feature_integration_tests/rust_test_scenarios/src/scenarios/mod.rs diff --git a/feature_integration_tests/rust_test_scenarios/src/tests/basic/only_shutdown_sequence.rs b/feature_integration_tests/rust_test_scenarios/src/tests/basic/only_shutdown_sequence.rs deleted file mode 100644 index 0ff1910bcd..0000000000 --- a/feature_integration_tests/rust_test_scenarios/src/tests/basic/only_shutdown_sequence.rs +++ /dev/null @@ -1,36 +0,0 @@ -// -// Copyright (c) 2025 Contributors to the Eclipse Foundation -// -// See the NOTICE file(s) distributed with this work for additional -// information regarding copyright ownership. -// -// This program and the accompanying materials are made available under the -// terms of the Apache License Version 2.0 which is available at -// -// -// SPDX-License-Identifier: Apache-2.0 -// -use crate::internals::runtime_helper::Runtime; -use test_scenarios_rust::scenario::Scenario; -use tracing::info; - -pub struct OnlyShutdownSequence; - -/// Checks (almost) empty program with only shutdown -impl Scenario for OnlyShutdownSequence { - fn name(&self) -> &str { - "only_shutdown" - } - - fn run(&self, input: &str) -> Result<(), String> { - let mut rt = Runtime::from_json(input)?.build(); - - rt.block_on(async move { - info!("Program entered engine"); - // TODO: Create a program with only shutdown sequence once it is supported. - info!("Program execution finished"); - }); - - Ok(()) - } -}