diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index adab2a94..b6a683a0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,91 +11,55 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Lint run: | make fmt + build: runs-on: deathstar steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: false # We'll handle submodules with smart caching - fetch-depth: 0 - - # - name: Cache bin - # id: cache-bin - # uses: actions/cache@v4 - # with: - # path: | - # bin - # key: bin-${{ hashFiles('Makefile') }}-v3 - # restore-keys: | - # bin-${{ hashFiles('Makefile') }}- - # bin- + - uses: actions/checkout@v4 - name: Download bin tools if: steps.cache-bin.outputs.cache-hit != 'true' run: | make download-bin - # - name: Cache submodules - # id: cache-submodules - # uses: actions/cache@v4 - # with: - # path: | - # lib/fprime - # lib/fprime-zephyr - # lib/zephyr-workspace/zephyr - # key: submodules-${{ hashFiles('.gitmodules') }}-v3 - # restore-keys: | - # submodules-${{ hashFiles('.gitmodules') }}- - # submodules- - - name: Setup submodules if: steps.cache-submodules.outputs.cache-hit != 'true' run: | make submodules - # - name: Cache python venv - # id: cache-python - # uses: actions/cache@v4 - # with: - # path: fprime-venv - # key: python-venv-${{ runner.os }}-${{ hashFiles('requirements.txt') }}-v3 - # restore-keys: | - # python-venv-${{ runner.os }}- - # python-venv- - - - name: Setup python venv - if: steps.cache-python.outputs.cache-hit != 'true' + - name: Create python venv run: | make fprime-venv - # - name: Cache Zephyr workspace and SDK - # id: cache-zephyr - # uses: actions/cache@v4 - # with: - # path: | - # lib/zephyr-workspace/modules - # lib/zephyr-workspace/bootloader - # ~/zephyr-sdk-0.17.2 - # key: zephyr-${{ hashFiles('west.yml') }}-${{ runner.os }}-v3 - # restore-keys: | - # zephyr-${{ hashFiles('west.yml') }}-${{ runner.os }}- - # zephyr- - - name: Setup Zephyr - if: steps.cache-zephyr.outputs.cache-hit != 'true' + if: steps.cache-zephyr-workspace.outputs.cache-hit != 'true' + run: | + make zephyr-workspace + + - name: Setup Zephyr SDK + if: steps.cache-zephyr-sdk.outputs.cache-hit != 'true' + run: | + make zephyr-sdk + + - name: Setup Zephyr Export run: | - make zephyr-setup - env: - PIP_DISABLE_PIP_VERSION_CHECK: 1 - PIP_NO_COMPILE: 1 + make zephyr-export + + - name: Install Zephyr Python Dependencies + run: | + make zephyr-python-deps + + - name: Generate + run: | + make generate - name: Build run: | - make generate-ci build-ci + make build - name: Upload build artifacts uses: actions/upload-artifact@v4 @@ -105,25 +69,32 @@ jobs: build-artifacts/zephyr.uf2 build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json retention-days: 30 + integration: - runs-on: [integration] + runs-on: + - integration needs: build steps: - uses: actions/checkout@v4 + - uses: actions/download-artifact@v5 + - name: Set up dependencies run: | - mkdir -p build-artifacts/zephyr/fprime-zephyr-deployment/dict && mv zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json build-artifacts/zephyr/fprime-zephyr-deployment/dict - make submodules - make fprime-venv + mkdir -p build-artifacts/zephyr/fprime-zephyr-deployment/dict \ + && mv zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json build-artifacts/zephyr/fprime-zephyr-deployment/dict + make submodules fprime-venv + - name: Trigger Bootloader run: | make bootloader sleep 10 + - name: Copy Firmware run: | picotool load ./zephyr.uf2 picotool reboot + - name: Run Integration Tests run: | make test-integration diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 794b0d8d..a9a96819 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,3 +33,14 @@ repos: - id: ruff-check args: [--fix, --select, I] # import sorting - id: ruff-format + +- repo: https://github.com/econchick/interrogate/ + rev: 1.7.0 + hooks: + - id: interrogate + args: + - --ignore-init-method + - --omit-covered-files + - --fail-under=100 + - -vv + - --color diff --git a/FprimeZephyrReference/Components/Burnwire/Burnwire.cpp b/FprimeZephyrReference/Components/Burnwire/Burnwire.cpp index a4536519..581678c6 100644 --- a/FprimeZephyrReference/Components/Burnwire/Burnwire.cpp +++ b/FprimeZephyrReference/Components/Burnwire/Burnwire.cpp @@ -72,13 +72,13 @@ void Burnwire ::schedIn_handler(FwIndexType portNum, U32 context) { // ---------------------------------------------------------------------- void Burnwire ::START_BURNWIRE_cmdHandler(FwOpcodeType opCode, U32 cmdSeq) { - this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); this->startBurn(); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); } void Burnwire ::STOP_BURNWIRE_cmdHandler(FwOpcodeType opCode, U32 cmdSeq) { - this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); this->stopBurn(); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); } } // namespace Components diff --git a/FprimeZephyrReference/test/bootloader_trigger.py b/FprimeZephyrReference/test/bootloader_trigger.py index c1e9c200..55a06771 100644 --- a/FprimeZephyrReference/test/bootloader_trigger.py +++ b/FprimeZephyrReference/test/bootloader_trigger.py @@ -1,13 +1,26 @@ +""" +bootloader_trigger.py: + +This module is responsible for sending the command to trigger the bootloader mode +on the PROVES hardware during integration tests. Once in bootloader mode, the device +can accept firmware updates. +""" + import os import subprocess import time import pytest from fprime_gds.common.testing_fw.api import IntegrationTestAPI +from int.common import cmdDispatch @pytest.fixture(scope="session", autouse=True) def start_gds(fprime_test_api_session: IntegrationTestAPI): + """Fixture to start GDS + + GDS is used to send the bootloader commands. + """ process = subprocess.Popen(["make", "gds-integration"], cwd=os.getcwd()) gds_working = False @@ -15,7 +28,7 @@ def start_gds(fprime_test_api_session: IntegrationTestAPI): while time.time() < timeout_time: try: fprime_test_api_session.send_and_assert_command( - command="CdhCore.cmdDisp.CMD_NO_OP" + command=f"{cmdDispatch}.CMD_NO_OP" ) gds_working = True break @@ -28,6 +41,10 @@ def start_gds(fprime_test_api_session: IntegrationTestAPI): def test_bootloader(fprime_test_api: IntegrationTestAPI): + """Trigger bootloader mode on PROVES hardware""" + # Don't use proves_send_and_assert_command here because we don't expect + # a response from the bootloader trigger command. The device will reboot + # into bootloader mode and may not send a command completion event. fprime_test_api.send_command( "ReferenceDeployment.bootloaderTrigger.TRIGGER_BOOTLOADER" ) diff --git a/FprimeZephyrReference/test/int/burnwire_test.py b/FprimeZephyrReference/test/int/burnwire_test.py index 882124b7..68e6808d 100644 --- a/FprimeZephyrReference/test/int/burnwire_test.py +++ b/FprimeZephyrReference/test/int/burnwire_test.py @@ -1,93 +1,64 @@ -# """ -# burnwire_test.py: +""" +burnwire_test.py: -# Integration tests for the Burnwire component. -# """ +Integration tests for the Burnwire component. +""" import pytest +from common import proves_send_and_assert_command from fprime_gds.common.testing_fw.api import IntegrationTestAPI -# Constants +burnwire = "ReferenceDeployment.burnwire" @pytest.fixture(autouse=True) -def reset_burnwire(fprime_test_api: IntegrationTestAPI): +def reset_burnwire(fprime_test_api: IntegrationTestAPI, start_gds): """Fixture to stop burnwire and clear histories before/after each test""" # Stop burnwire and clear before test - fprime_test_api.clear_histories() stop_burnwire(fprime_test_api) yield # Clear again after test to prevent residue - fprime_test_api.clear_histories() stop_burnwire(fprime_test_api) def stop_burnwire(fprime_test_api: IntegrationTestAPI): - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.burnwire.STOP_BURNWIRE" - ) + """Stop the burnwire and clear histories""" + proves_send_and_assert_command(fprime_test_api, f"{burnwire}.STOP_BURNWIRE") - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.SetBurnwireState", "OFF", timeout=10 - ) + fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "OFF", timeout=10) - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.BurnwireEndCount", timeout=2 - ) - - received_events = fprime_test_api.get_event_subhistory() - print(f"Received events: {received_events}") + fprime_test_api.assert_event(f"{burnwire}.BurnwireEndCount", timeout=2) def test_01_start_and_stop_burnwire(fprime_test_api: IntegrationTestAPI, start_gds): """Test that burnwire starts and stops as expected""" # Start burnwire - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.burnwire.START_BURNWIRE" - ) + proves_send_and_assert_command(fprime_test_api, f"{burnwire}.START_BURNWIRE") # Wait for SetBurnwireState = ON - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.SetBurnwireState", "ON", timeout=2 - ) + fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "ON", timeout=2) - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.SafetyTimerState", timeout=2 - ) + fprime_test_api.assert_event(f"{burnwire}.SafetyTimerState", timeout=2) - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.SetBurnwireState", "OFF", timeout=10 - ) + fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "OFF", timeout=10) - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.BurnwireEndCount", timeout=2 - ) + fprime_test_api.assert_event(f"{burnwire}.BurnwireEndCount", timeout=2) def test_02_manual_stop_before_timeout(fprime_test_api: IntegrationTestAPI, start_gds): """Test that burnwire stops manually before the safety timer expires""" # Start burnwire - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.burnwire.START_BURNWIRE" - ) + proves_send_and_assert_command(fprime_test_api, f"{burnwire}.START_BURNWIRE") # Confirm Burnwire turned ON - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.SetBurnwireState", "ON", timeout=2 - ) + fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "ON", timeout=2) # # Stop burnwire before safety timer triggers - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.burnwire.STOP_BURNWIRE" - ) + proves_send_and_assert_command(fprime_test_api, f"{burnwire}.STOP_BURNWIRE") # Confirm Burnwire turned OFF - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.SetBurnwireState", "OFF", timeout=2 - ) + fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "OFF", timeout=2) - fprime_test_api.assert_event( - "ReferenceDeployment.burnwire.BurnwireEndCount", timeout=2 - ) + fprime_test_api.assert_event(f"{burnwire}.BurnwireEndCount", timeout=2) diff --git a/FprimeZephyrReference/test/int/common.py b/FprimeZephyrReference/test/int/common.py new file mode 100644 index 00000000..a16a583e --- /dev/null +++ b/FprimeZephyrReference/test/int/common.py @@ -0,0 +1,42 @@ +""" +common.py: + +This module provides a functions and constants shared by +integration tests for PROVES microcontroller hardware. +""" + +from fprime_gds.common.testing_fw.api import IntegrationTestAPI +from fprime_gds.common.testing_fw.predicates import event_predicate + +cmdDispatch = "CdhCore.cmdDisp" + + +def proves_send_and_assert_command( + fprime_test_api: IntegrationTestAPI, + command: str, + args: list[str] = [], + events: list[event_predicate] = [], +): + """Send command and assert completion + + PROVES microcontroller hardware responds more slowly than typical FPrime + hardware which use microprocessors. As a result, some commands may + take longer to complete. This function clears histories before sending + the command, sets a longer timeout for command completion, and retries + up to 3 times if command assertion fails. + """ + fprime_test_api.clear_histories() + + for attempt in range(3): + try: + fprime_test_api.send_and_assert_command( + command, + args, + timeout=5, + max_delay=5, + events=events, + ) + break + except AssertionError: + if attempt == 2: + raise diff --git a/FprimeZephyrReference/test/int/conftest.py b/FprimeZephyrReference/test/int/conftest.py index bc8d891c..6b1b8540 100644 --- a/FprimeZephyrReference/test/int/conftest.py +++ b/FprimeZephyrReference/test/int/conftest.py @@ -1,14 +1,25 @@ +""" +conftest.py: + +Pytest configuration for integration tests. +""" + import os import signal import subprocess import time import pytest +from common import cmdDispatch from fprime_gds.common.testing_fw.api import IntegrationTestAPI @pytest.fixture(scope="session") def start_gds(fprime_test_api_session: IntegrationTestAPI): + """Fixture to start GDS before tests and stop after tests + + GDS is used to send commands and receive telemetry/events. + """ pro = subprocess.Popen( ["make", "gds-integration"], cwd=os.getcwd(), @@ -21,7 +32,7 @@ def start_gds(fprime_test_api_session: IntegrationTestAPI): while time.time() < timeout_time: try: fprime_test_api_session.send_and_assert_command( - command="CdhCore.cmdDisp.CMD_NO_OP" + command=f"{cmdDispatch}.CMD_NO_OP" ) gds_working = True break diff --git a/FprimeZephyrReference/test/int/imu_manager_test.py b/FprimeZephyrReference/test/int/imu_manager_test.py index 2a84891f..d4db21bf 100644 --- a/FprimeZephyrReference/test/int/imu_manager_test.py +++ b/FprimeZephyrReference/test/int/imu_manager_test.py @@ -5,23 +5,28 @@ """ import pytest +from common import proves_send_and_assert_command from fprime_gds.common.data_types.ch_data import ChData from fprime_gds.common.testing_fw.api import IntegrationTestAPI +lsm6dsoManager = "ReferenceDeployment.lsm6dsoManager" +lis2mdlManager = "ReferenceDeployment.lis2mdlManager" + @pytest.fixture(autouse=True) -def send_packet(fprime_test_api: IntegrationTestAPI): +def send_packet(fprime_test_api: IntegrationTestAPI, start_gds): """Fixture to clear histories and send the IMU packet before each test""" - fprime_test_api.clear_histories() - fprime_test_api.send_and_assert_command( - "CdhCore.tlmSend.SEND_PKT", ["6"], max_delay=2 + proves_send_and_assert_command( + fprime_test_api, + "CdhCore.tlmSend.SEND_PKT", + ["6"], ) def test_01_acceleration_telemetry(fprime_test_api: IntegrationTestAPI, start_gds): """Test that we can get Acceleration telemetry""" result: ChData = fprime_test_api.assert_telemetry( - "ReferenceDeployment.lsm6dsoManager.Acceleration", start="NOW", timeout=3 + f"{lsm6dsoManager}.Acceleration", start="NOW", timeout=3 ) reading: dict[float] = result.get_val() @@ -33,7 +38,7 @@ def test_01_acceleration_telemetry(fprime_test_api: IntegrationTestAPI, start_gd def test_02_angular_velocity_telemetry(fprime_test_api: IntegrationTestAPI, start_gds): """Test that we can get AngularVelocity telemetry""" result: ChData = fprime_test_api.assert_telemetry( - "ReferenceDeployment.lsm6dsoManager.AngularVelocity", start="NOW", timeout=3 + f"{lsm6dsoManager}.AngularVelocity", start="NOW", timeout=3 ) reading: dict[float] = result.get_val() @@ -45,7 +50,7 @@ def test_02_angular_velocity_telemetry(fprime_test_api: IntegrationTestAPI, star def test_03_temperature_telemetry(fprime_test_api: IntegrationTestAPI, start_gds): """Test that we can get Temperature telemetry""" result: ChData = fprime_test_api.assert_telemetry( - "ReferenceDeployment.lsm6dsoManager.Temperature", start="NOW", timeout=3 + f"{lsm6dsoManager}.Temperature", start="NOW", timeout=3 ) reading: int = result.get_val() @@ -55,7 +60,7 @@ def test_03_temperature_telemetry(fprime_test_api: IntegrationTestAPI, start_gds def test_04_magnetic_field_telemetry(fprime_test_api: IntegrationTestAPI, start_gds): """Test that we can get MagneticField telemetry""" result: ChData = fprime_test_api.assert_telemetry( - "ReferenceDeployment.lis2mdlManager.MagneticField", start="NOW", timeout=3 + f"{lis2mdlManager}.MagneticField", start="NOW", timeout=3 ) reading: dict[float] = result.get_val() diff --git a/FprimeZephyrReference/test/int/rtc_test.py b/FprimeZephyrReference/test/int/rtc_test.py index a16c51ec..76ef3e85 100644 --- a/FprimeZephyrReference/test/int/rtc_test.py +++ b/FprimeZephyrReference/test/int/rtc_test.py @@ -9,15 +9,19 @@ from datetime import datetime, timezone import pytest +from common import cmdDispatch, proves_send_and_assert_command from fprime.common.models.serialize.numerical_types import U32Type from fprime.common.models.serialize.time_type import TimeType from fprime_gds.common.data_types.ch_data import ChData from fprime_gds.common.data_types.event_data import EventData from fprime_gds.common.testing_fw.api import IntegrationTestAPI +from fprime_gds.common.testing_fw.predicates import event_predicate + +rtcManager = "ReferenceDeployment.rtcManager" @pytest.fixture(autouse=True) -def set_now_time(fprime_test_api: IntegrationTestAPI): +def set_now_time(fprime_test_api: IntegrationTestAPI, start_gds): """Fixture to set the time to test runner's time after each test""" yield set_time(fprime_test_api) @@ -38,14 +42,13 @@ def set_time(fprime_test_api: IntegrationTestAPI, dt: datetime = None): Second=dt.second, ) time_data_str = json.dumps(time_data) - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.rtcManager.TIME_SET", + proves_send_and_assert_command( + fprime_test_api, + f"{rtcManager}.TIME_SET", [ time_data_str, ], - max_delay=2, ) - fprime_test_api.assert_event("ReferenceDeployment.rtcManager.TimeSet", timeout=2) def test_01_time_set(fprime_test_api: IntegrationTestAPI, start_gds): @@ -56,9 +59,7 @@ def test_01_time_set(fprime_test_api: IntegrationTestAPI, start_gds): set_time(fprime_test_api, curiosity_landing) # Fetch event data - result: EventData = fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.TimeSet", timeout=2 - ) + result: EventData = fprime_test_api.assert_event(f"{rtcManager}.TimeSet", timeout=2) # Fetch previously set time from event args event_previous_time_arg: U32Type = result.args[0] @@ -83,9 +84,7 @@ def test_01_time_set(fprime_test_api: IntegrationTestAPI, start_gds): pytest.approx(event_time, abs=30) == curiosity_landing # Fetch event data - result: EventData = fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.TimeSet", timeout=2 - ) + result: EventData = fprime_test_api.assert_event(f"{rtcManager}.TimeSet", timeout=2) # Assert time is within 30 seconds of now pytest.approx(event_time, abs=30) == datetime.now(timezone.utc) @@ -96,7 +95,7 @@ def test_02_time_incrementing(fprime_test_api: IntegrationTestAPI, start_gds): # Fetch initial time result: ChData = fprime_test_api.assert_telemetry( - "CdhCore.cmdDisp.CommandsDispatched", timeout=3 + f"{cmdDispatch}.CommandsDispatched", timeout=3 ) # Convert FPrime time to datetime @@ -109,7 +108,7 @@ def test_02_time_incrementing(fprime_test_api: IntegrationTestAPI, start_gds): # Fetch updated time result: ChData = fprime_test_api.assert_telemetry( - "CdhCore.cmdDisp.CommandsDispatched", timeout=3 + f"{cmdDispatch}.CommandsDispatched", timeout=3 ) # Convert FPrime time to datetime @@ -125,6 +124,19 @@ def test_02_time_incrementing(fprime_test_api: IntegrationTestAPI, start_gds): def test_03_time_not_set_event(fprime_test_api: IntegrationTestAPI, start_gds): """Test that a TimeNotSet event is emitted when setting time with invalid data""" + # List of events we expect to see + events: list[event_predicate] = [ + fprime_test_api.get_event_pred(f"{rtcManager}.YearValidationFailed"), + fprime_test_api.get_event_pred(f"{rtcManager}.MonthValidationFailed"), + fprime_test_api.get_event_pred(f"{rtcManager}.DayValidationFailed"), + fprime_test_api.get_event_pred(f"{rtcManager}.HourValidationFailed"), + fprime_test_api.get_event_pred(f"{rtcManager}.MinuteValidationFailed"), + fprime_test_api.get_event_pred(f"{rtcManager}.SecondValidationFailed"), + fprime_test_api.get_event_pred(f"{rtcManager}.TimeNotSet"), + fprime_test_api.get_event_pred(f"{cmdDispatch}.OpCodeDispatched"), + fprime_test_api.get_event_pred(f"{cmdDispatch}.OpCodeError"), + ] + # Clear histories fprime_test_api.clear_histories() @@ -138,40 +150,6 @@ def test_03_time_not_set_event(fprime_test_api: IntegrationTestAPI, start_gds): Second=12345, ) time_data_str = json.dumps(time_data) - fprime_test_api.send_command( - "ReferenceDeployment.rtcManager.TIME_SET", - [ - time_data_str, - ], - ) - - # Assert time not set event is emitted - fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.YearValidationFailed", timeout=2 - ) - - fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.MonthValidationFailed", timeout=2 + fprime_test_api.send_and_assert_event( + f"{rtcManager}.TIME_SET", [time_data_str], events, timeout=10 ) - - fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.DayValidationFailed", timeout=2 - ) - - fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.HourValidationFailed", timeout=2 - ) - - fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.MinuteValidationFailed", timeout=2 - ) - - fprime_test_api.assert_event( - "ReferenceDeployment.rtcManager.SecondValidationFailed", timeout=2 - ) - - fprime_test_api.assert_event("ReferenceDeployment.rtcManager.TimeNotSet", timeout=2) - - fprime_test_api.assert_event("CdhCore.cmdDisp.OpCodeDispatched", timeout=2) - - fprime_test_api.assert_event("CdhCore.cmdDisp.OpCodeError", timeout=2) diff --git a/FprimeZephyrReference/test/int/watchdog_test.py b/FprimeZephyrReference/test/int/watchdog_test.py index b271d04a..02e984d2 100644 --- a/FprimeZephyrReference/test/int/watchdog_test.py +++ b/FprimeZephyrReference/test/int/watchdog_test.py @@ -7,12 +7,15 @@ import time import pytest +from common import proves_send_and_assert_command from fprime_gds.common.data_types.ch_data import ChData from fprime_gds.common.testing_fw.api import IntegrationTestAPI +watchdog = "ReferenceDeployment.watchdog" + @pytest.fixture(autouse=True) -def ensure_watchdog_running(fprime_test_api: IntegrationTestAPI): +def ensure_watchdog_running(fprime_test_api: IntegrationTestAPI, start_gds): """Fixture to ensure watchdog is started before and after each test""" start_watchdog(fprime_test_api) yield @@ -21,22 +24,21 @@ def ensure_watchdog_running(fprime_test_api: IntegrationTestAPI): def start_watchdog(fprime_test_api: IntegrationTestAPI): """Helper function to start the watchdog""" - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.watchdog.START_WATCHDOG", max_delay=2 - ) - fprime_test_api.assert_event( - "ReferenceDeployment.watchdog.WatchdogStart", timeout=2 + proves_send_and_assert_command( + fprime_test_api, + f"{watchdog}.START_WATCHDOG", ) def get_watchdog_transitions(fprime_test_api: IntegrationTestAPI) -> int: """Helper function to request packet and get fresh WatchdogTransitions telemetry""" - fprime_test_api.clear_histories() - fprime_test_api.send_and_assert_command( - "CdhCore.tlmSend.SEND_PKT", ["5"], max_delay=2 + proves_send_and_assert_command( + fprime_test_api, + "CdhCore.tlmSend.SEND_PKT", + ["5"], ) result: ChData = fprime_test_api.assert_telemetry( - "ReferenceDeployment.watchdog.WatchdogTransitions", start="NOW", timeout=3 + f"{watchdog}.WatchdogTransitions", start="NOW", timeout=3 ) return result.get_val() @@ -64,11 +66,11 @@ def test_03_stop_watchdog_command(fprime_test_api: IntegrationTestAPI, start_gds Test STOP_WATCHDOG command sends and emits WatchdogStop event and WatchdogTransitions stops incrementing """ - fprime_test_api.clear_histories() # Send stop command - fprime_test_api.send_and_assert_command( - "ReferenceDeployment.watchdog.STOP_WATCHDOG", max_delay=2 + proves_send_and_assert_command( + fprime_test_api, + f"{watchdog}.STOP_WATCHDOG", ) # Check for watchdog stop event diff --git a/Makefile b/Makefile index 64fb2f45..c27a53e8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: all -all: submodules fprime-venv zephyr-setup generate-if-needed build +all: submodules fprime-venv zephyr generate-if-needed build .PHONY: help help: ## Display this help. @@ -9,31 +9,17 @@ help: ## Display this help. .PHONY: submodules submodules: ## Initialize and update git submodules - @echo "Initializing and updating git submodules..." - git submodule update --init --recursive + @git submodule update --init --recursive export VIRTUAL_ENV ?= $(shell pwd)/fprime-venv -fprime-venv: ## Create a virtual environment - @$(MAKE) uv - @echo "Creating virtual environment..." - @$(UV) venv fprime-venv - @$(UV) pip install --prerelease=allow --requirement requirements.txt +.PHONY: fprime-venv +fprime-venv: uv ## Create a virtual environment + @$(UV) venv fprime-venv --allow-existing + @$(UV) pip install --prerelease=allow --requirement requirements.txt patch-gps-package: cp custom_space_data_link.py fprime-venv/lib/python3.13/site-packages/fprime_gds/common/communication/ccsds/space_data_link.py -.PHONY: zephyr-setup -zephyr-setup: fprime-venv ## Set up Zephyr environment - @test -d lib/zephyr-workspace/modules/hal/rpi_pico || test -d ../lib/zephyr-workspace/modules/hal/rpi_pico || { \ - echo "Setting up Zephyr environment..."; \ - rm -rf ../.west/ && \ - $(UVX) west init --local . && \ - $(UVX) west update && \ - $(UVX) west zephyr-export && \ - $(UV) run west packages pip --install && \ - $(UV) run west sdk install --toolchains arm-zephyr-eabi; \ - } - ##@ Development .PHONY: pre-commit-install @@ -45,31 +31,21 @@ fmt: pre-commit-install ## Lint and format files @$(UVX) pre-commit run --all-files .PHONY: generate -generate: submodules fprime-venv zephyr-setup ## Generate FPrime-Zephyr Proves Core Reference - @echo "Generating FPrime-Zephyr Proves Core Reference..." - @$(UV) run fprime-util generate --force - -.PHONY: generate-ci -generate-ci: - @$(UV) run fprime-util generate --force +generate: submodules fprime-venv zephyr ## Generate FPrime-Zephyr Proves Core Reference + @$(UV_RUN) fprime-util generate --force .PHONY: generate-if-needed BUILD_DIR ?= $(shell pwd)/build-fprime-automatic-zephyr generate-if-needed: - @test -s $(BUILD_DIR) || $(MAKE) generate + @test -d $(BUILD_DIR) || $(MAKE) generate .PHONY: build -build: submodules zephyr-setup fprime-venv generate-if-needed ## Build FPrime-Zephyr Proves Core Reference - @echo "Building..." - @$(UV) run fprime-util build - -.PHONY: build-ci -build-ci: - @$(UV) run fprime-util build +build: submodules zephyr fprime-venv generate-if-needed ## Build FPrime-Zephyr Proves Core Reference + @$(UV_RUN) fprime-util build .PHONY: test-integration test-integration: uv - @$(UV) run pytest FprimeZephyrReference/test/int --deployment build-artifacts/zephyr/fprime-zephyr-deployment + @$(UV_RUN) pytest FprimeZephyrReference/test/int --deployment build-artifacts/zephyr/fprime-zephyr-deployment .PHONY: bootloader bootloader: uv @@ -77,26 +53,16 @@ bootloader: uv echo "RP2350 already in bootloader mode - skipping trigger"; \ else \ echo "RP2350 not in bootloader mode - triggering bootloader"; \ - $(UV) run pytest FprimeZephyrReference/test/bootloader_trigger.py --deployment build-artifacts/zephyr/fprime-zephyr-deployment; \ + $(UV_RUN) pytest FprimeZephyrReference/test/bootloader_trigger.py --deployment build-artifacts/zephyr/fprime-zephyr-deployment; \ fi .PHONY: clean clean: ## Remove all gitignored files git clean -dfX -.PHONY: clean-zephyr -clean-zephyr: ## Remove all Zephyr build files - rm -rf lib/zephyr-workspace/bootloader lib/zephyr-workspace/modules lib/zephyr-workspace/tools - -.PHONY: clean-zephyr-sdk -clean-zephyr-sdk: ## Remove Zephyr SDK (reinstall with 'make zephyr-setup') - @echo "Removing Zephyr SDK..." - rm -rf ~/zephyr-sdk-* - @echo "Run 'make zephyr-setup' to reinstall with minimal ARM-only toolchain" - ##@ Operations -GDS_COMMAND ?= $(UV) run fprime-gds -n --dictionary $(ARTIFACT_DIR)/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json --communication-selection uart --uart-baud 115200 --output-unframed-data +GDS_COMMAND ?= $(UV_RUN) fprime-gds -n --dictionary $(ARTIFACT_DIR)/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json --communication-selection uart --uart-baud 115200 --output-unframed-data ARTIFACT_DIR ?= $(shell pwd)/build-artifacts .PHONY: gds @@ -108,22 +74,6 @@ gds: ## Run FPrime GDS gds-integration: @$(GDS_COMMAND) --gui=none -##@ Build Tools - -.PHONY: download-bin -download-bin: uv - -BIN_DIR ?= $(shell pwd)/bin -$(BIN_DIR): - mkdir -p $(BIN_DIR) - -### Tool Versions -UV_VERSION ?= 0.8.13 - -UV_DIR ?= $(BIN_DIR)/uv-$(UV_VERSION) -UV ?= $(UV_DIR)/uv -UVX ?= $(UV_DIR)/uvx -.PHONY: uv -uv: $(UV) ## Download uv -$(UV): $(BIN_DIR) - @test -s $(UV) || { mkdir -p $(UV_DIR); curl -LsSf https://astral.sh/uv/$(UV_VERSION)/install.sh | UV_INSTALL_DIR=$(UV_DIR) sh > /dev/null; } +include lib/makelib/build-tools.mk +include lib/makelib/ci.mk +include lib/makelib/zephyr.mk diff --git a/circuit-python-lora-passthrough/boot.py b/circuit-python-lora-passthrough/boot.py index 6776ff30..609f9096 100644 --- a/circuit-python-lora-passthrough/boot.py +++ b/circuit-python-lora-passthrough/boot.py @@ -1,3 +1,9 @@ +""" +boot.py + +TODO(nateinaction): I don't know what this file is for yet. +""" + import usb_cdc usb_cdc.enable(console=True, data=True) diff --git a/lib/makelib/build-tools.mk b/lib/makelib/build-tools.mk new file mode 100644 index 00000000..5de801aa --- /dev/null +++ b/lib/makelib/build-tools.mk @@ -0,0 +1,22 @@ +##@ Build Tools + +.PHONY: download-bin +download-bin: uv + +TOOLS_DIR ?= $(shell pwd)/bin +$(TOOLS_DIR): + mkdir -p $(TOOLS_DIR) + +### Tool Versions +UV_VERSION ?= 0.8.13 + +### uv & uvx +UV_DIR ?= $(TOOLS_DIR)/uv-$(UV_VERSION) +UV ?= $(UV_DIR)/uv +UVX ?= $(UV_DIR)/uvx +.PHONY: uv +uv: $(UV) ## Download uv +$(UV): $(TOOLS_DIR) + @test -s $(UV) || { mkdir -p $(UV_DIR); curl -LsSf https://astral.sh/uv/$(UV_VERSION)/install.sh | UV_INSTALL_DIR=$(UV_DIR) sh > /dev/null; } + +UV_RUN ?= $(UV) run --active diff --git a/lib/makelib/ci.mk b/lib/makelib/ci.mk new file mode 100644 index 00000000..161b8fc8 --- /dev/null +++ b/lib/makelib/ci.mk @@ -0,0 +1,3 @@ +.PHONY: minimize-uv-cache +minimize-uv-cache: + @$(UV) cache prune --ci diff --git a/lib/makelib/zephyr.mk b/lib/makelib/zephyr.mk new file mode 100644 index 00000000..fa292659 --- /dev/null +++ b/lib/makelib/zephyr.mk @@ -0,0 +1,65 @@ +##@ Zephyr + +# UV runs west with the active virtual environment +WEST ?= $(UV_RUN) west + +# UVX runs west without the active virtual environment +WESTX ?= $(UVX) west + +.PHONY: zephyr +zephyr: zephyr-config zephyr-workspace zephyr-export zephyr-sdk zephyr-python-deps + +.PHONY: clean-zephyr +clean-zephyr: clean-zephyr-config clean-zephyr-workspace clean-zephyr-export clean-zephyr-sdk + +.PHONY: zephyr-config +zephyr-config: fprime-venv ## Configure west + @test -f ../.west/config || { \ + $(WEST) init --local .; \ + } + +.PHONY: clean-zephyr-config +clean-zephyr-config: ## Remove west configuration + rm -rf ../.west + +.PHONY: zephyr-workspace +zephyr-workspace: fprime-venv ## Setup Zephyr bootloader, modules, and tools directories + @test -d ../lib/zephyr-workspace/bootloader || \ + test -d ../lib/zephyr-workspace/modules || \ + test -d ../lib/zephyr-workspace/tools || { \ + $(WESTX) update; \ + } + +.PHONY: clean-zephyr-workspace +clean-zephyr-workspace: ## Remove Zephyr bootloader, modules, and tools directories + rm -rf ../lib + +CMAKE_PACKAGES ?= ~/.cmake/packages +.PHONY: zephyr-export +zephyr-export: fprime-venv ## Export Zephyr environment variables + @test -d $(CMAKE_PACKAGES)/Zephyr/ || \ + test -d $(CMAKE_PACKAGES)/ZephyrUnittest/ || \ + { \ + $(WESTX) zephyr-export; \ + } + +.PHONY: clean-zephyr-export +clean-zephyr-export: ## Remove Zephyr exported files + rm -rf $(CMAKE_PACKAGES)/Zephyr $(CMAKE_PACKAGES)/ZephyrUnittest/ + +ZEPHYR_PATH ?= lib/zephyr-workspace/zephyr +SDK_VERSION ?= $(shell cat $(ZEPHYR_PATH)/SDK_VERSION) +ZEPHYR_SDK_PATH ?= ~/zephyr-sdk-$(SDK_VERSION) +.PHONY: zephyr-sdk +zephyr-sdk: fprime-venv ## Install Zephyr SDK + @test -d $(ZEPHYR_SDK_PATH) || { \ + $(WEST) sdk install --toolchains arm-zephyr-eabi; \ + } + +.PHONY: clean-zephyr-sdk +clean-zephyr-sdk: ## Remove Zephyr SDK + rm -rf $(ZEPHYR_SDK_PATH) + +.PHONY: zephyr-python-deps +zephyr-python-deps: fprime-venv ## Install Zephyr Python dependencies + @$(WEST) uv pip --install -- --prerelease=allow --quiet diff --git a/lib/west-commands/uv.py b/lib/west-commands/uv.py new file mode 100644 index 00000000..44e1be0e --- /dev/null +++ b/lib/west-commands/uv.py @@ -0,0 +1,197 @@ +# Copyright (c) 2024 Basalte bv +# +# SPDX-License-Identifier: Apache-2.0 +""" +uv.py + +West command to manage packages for Zephyr and its modules using UV. +""" + +import argparse +import os +import subprocess +import sys +import textwrap +from itertools import chain +from pathlib import Path + +from west.commands import WestCommand + +sys.path.append( + os.fspath( + Path(__file__).parent.parent + / "zephyr-workspace" + / "zephyr" + / "scripts" + / "west_commands" + ) +) +from zephyr_ext_common import ZEPHYR_BASE + +sys.path.append(os.fspath(ZEPHYR_BASE / "scripts")) +import zephyr_module + + +def in_venv() -> bool: + """Check if we are running inside a virtual environment.""" + return sys.prefix != sys.base_prefix + + +class Uv(WestCommand): + """Provide interface to use UV through Zephyr's west build tool.""" + + def __init__(self): + super().__init__( + "uv", + "manage packages for Zephyr", + "List and Install packages for Zephyr and modules", + accepts_unknown_args=True, + ) + + def do_add_parser(self, parser_adder): + """Add argument parser for this command.""" + parser = parser_adder.add_parser( + self.name, + help=self.help, + description=self.description, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( + """ + Listing uv: + + Run 'west uv ' to list all dependencies + available from a given package manager, already + installed and not. These can be filtered by module, + see 'west uv --help' for details. + """ + ), + ) + + parser.add_argument( + "-m", + "--module", + action="append", + default=[], + dest="modules", + metavar="", + help="Zephyr module to run the 'packages' command for. " + "Use 'zephyr' if the 'packages' command should run for Zephyr itself. " + "Option can be passed multiple times. " + "If this option is not given, the 'packages' command will run for Zephyr " + "and all modules.", + ) + + subparsers_gen = parser.add_subparsers( + metavar="", + dest="manager", + help="select a manager.", + required=True, + ) + + pip_parser = subparsers_gen.add_parser( + "pip", + help="manage pip packages", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent( + """ + Manage pip packages: + + Run 'west uv pip' to print all requirement files needed by + Zephyr and modules. + + The output is compatible with the requirements file format itself. + """ + ), + ) + + pip_parser.add_argument( + "--install", + action="store_true", + help="Install pip requirements instead of listing them. " + "A single 'pip install' command is built and executed. " + "Additional pip arguments can be passed after a -- separator " + "from the original 'west uv pip --install' command. For example pass " + "'--dry-run' to pip not to actually install anything, but print what would be.", + ) + + pip_parser.add_argument( + "--ignore-venv-check", + action="store_true", + help="Ignore the virtual environment check. " + "This is useful when running 'west uv pip --install' " + "in a CI environment where the virtual environment is not set up.", + ) + + return parser + + def do_run(self, args, unknown): + """Execute the command.""" + if len(unknown) > 0 and unknown[0] != "--": + self.die( + f'Unknown argument "{unknown[0]}"; ' + 'arguments for the manager should be passed after "--"' + ) + + # Store the zephyr modules for easier access + self.zephyr_modules = zephyr_module.parse_modules(ZEPHYR_BASE, self.manifest) + + if args.modules: + # Check for unknown module names + module_names = [m.meta.get("name") for m in self.zephyr_modules] + module_names.append("zephyr") + for m in args.modules: + if m not in module_names: + self.die(f'Unknown zephyr module "{m}"') + + if args.manager == "pip": + return self.do_run_pip(args, unknown[1:]) + + # Unreachable but print an error message if an implementation is missing. + self.die(f'Unsupported package manager: "{args.manager}"') + + def do_run_pip(self, args, manager_args): + """Execute the uv pip install subcommand.""" + requirements = [] + + if not args.modules or "zephyr" in args.modules: + requirements.append(ZEPHYR_BASE / "scripts/requirements.txt") + + for module in self.zephyr_modules: + module_name = module.meta.get("name") + if args.modules and module_name not in args.modules: + if args.install: + self.dbg(f"Skipping module {module_name}") + continue + + # Get the optional pip section from the package managers + pip = module.meta.get("package-managers", {}).get("pip") + if pip is None: + if args.install: + self.dbg(f"Nothing to install for {module_name}") + continue + + # Add requirements files + requirements += [ + Path(module.project) / r for r in pip.get("requirement-files", []) + ] + + if args.install: + if not in_venv() and not args.ignore_venv_check: + self.die("Running pip install outside of a virtual environment") + + if len(requirements) > 0: + subprocess.check_call( + ["uv", "pip", "install"] + + list(chain.from_iterable([("-r", r) for r in requirements])) + + manager_args + ) + else: + self.inf("Nothing to install") + return + + if len(manager_args) > 0: + self.die( + f'west uv pip does not support unknown arguments: "{manager_args}"' + ) + + self.inf("\n".join([f"-r {r}" for r in requirements])) diff --git a/requirements.txt b/requirements.txt index b3b08b46..f6c91b4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,4 @@ -# Base on: -# -r https://raw.githubusercontent.com/nasa/fprime/refs/tags/v4.0.0a1/requirements.txt --r lib/fprime/requirements.txt - pip west -# Requirements files needed for building Zephyr -# Zephyr workflow requirements removed due to conflicts with F Prime --r lib/zephyr-workspace/zephyr/scripts/requirements-base.txt --r lib/zephyr-workspace/zephyr/scripts/requirements-build-test.txt --r lib/zephyr-workspace/zephyr/scripts/requirements-run-test.txt + +-r lib/fprime/requirements.txt diff --git a/west-commands.yml b/west-commands.yml new file mode 100644 index 00000000..bf4f4e53 --- /dev/null +++ b/west-commands.yml @@ -0,0 +1,7 @@ +# Keep the help strings in sync with the values in the .py files! +west-commands: + - file: lib/west-commands/uv.py + commands: + - name: uv + class: Uv + help: manage packages for Zephyr with uv diff --git a/west.yml b/west.yml index 1c8bfc36..5e9079dc 100644 --- a/west.yml +++ b/west.yml @@ -86,3 +86,4 @@ manifest: self: path: . + west-commands: west-commands.yml