Improve Zephyr environment management + improve integration test reliability

.github/workflows/ci.yaml

@@ -11,91 +11,55 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
       - name: Lint
         run: |
           make fmt
+
   build:
     runs-on: deathstar
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          submodules: false # We'll handle submodules with smart caching
-          fetch-depth: 0
-
-      # - name: Cache bin
-      #   id: cache-bin
-      #   uses: actions/cache@v4
-      #   with:
-      #     path: |
-      #       bin
-      #     key: bin-${{ hashFiles('Makefile') }}-v3
-      #     restore-keys: |
-      #       bin-${{ hashFiles('Makefile') }}-
-      #       bin-
+      - uses: actions/checkout@v4
 
       - name: Download bin tools
         if: steps.cache-bin.outputs.cache-hit != 'true'
         run: |
           make download-bin
 
-      # - name: Cache submodules
-      #   id: cache-submodules
-      #   uses: actions/cache@v4
-      #   with:
-      #     path: |
-      #       lib/fprime
-      #       lib/fprime-zephyr
-      #       lib/zephyr-workspace/zephyr
-      #     key: submodules-${{ hashFiles('.gitmodules') }}-v3
-      #     restore-keys: |
-      #       submodules-${{ hashFiles('.gitmodules') }}-
-      #       submodules-
-
       - name: Setup submodules
         if: steps.cache-submodules.outputs.cache-hit != 'true'
         run: |
           make submodules
 
-      # - name: Cache python venv
-      #   id: cache-python
-      #   uses: actions/cache@v4
-      #   with:
-      #     path: fprime-venv
-      #     key: python-venv-${{ runner.os }}-${{ hashFiles('requirements.txt') }}-v3
-      #     restore-keys: |
-      #       python-venv-${{ runner.os }}-
-      #       python-venv-
-
-      - name: Setup python venv
-        if: steps.cache-python.outputs.cache-hit != 'true'
+      - name: Create python venv
         run: |
           make fprime-venv
 
-      # - name: Cache Zephyr workspace and SDK
-      #   id: cache-zephyr
-      #   uses: actions/cache@v4
-      #   with:
-      #     path: |
-      #       lib/zephyr-workspace/modules
-      #       lib/zephyr-workspace/bootloader
-      #       ~/zephyr-sdk-0.17.2
-      #     key: zephyr-${{ hashFiles('west.yml') }}-${{ runner.os }}-v3
-      #     restore-keys: |
-      #       zephyr-${{ hashFiles('west.yml') }}-${{ runner.os }}-
-      #       zephyr-
-
       - name: Setup Zephyr
-        if: steps.cache-zephyr.outputs.cache-hit != 'true'
+        if: steps.cache-zephyr-workspace.outputs.cache-hit != 'true'
+        run: |
+          make zephyr-workspace
+
+      - name: Setup Zephyr SDK
+        if: steps.cache-zephyr-sdk.outputs.cache-hit != 'true'
+        run: |
+          make zephyr-sdk
+
+      - name: Setup Zephyr Export
         run: |
-          make zephyr-setup
-        env:
-          PIP_DISABLE_PIP_VERSION_CHECK: 1
-          PIP_NO_COMPILE: 1
+          make zephyr-export
+
+      - name: Install Zephyr Python Dependencies
+        run: |
+          make zephyr-python-deps
+
+      - name: Generate
+        run: |
+          make generate
 
       - name: Build
         run: |
-          make generate-ci build-ci
+          make build
 
       - name: Upload build artifacts
         uses: actions/upload-artifact@v4
@@ -105,25 +69,32 @@ jobs:
             build-artifacts/zephyr.uf2
             build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json
           retention-days: 30
+
   integration:
-    runs-on: [integration]
+    runs-on:
+      - integration
     needs: build
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/download-artifact@v5
+
       - name: Set up dependencies
         run: |
-          mkdir -p build-artifacts/zephyr/fprime-zephyr-deployment/dict && mv zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json build-artifacts/zephyr/fprime-zephyr-deployment/dict
-            make submodules
-            make fprime-venv
+          mkdir -p build-artifacts/zephyr/fprime-zephyr-deployment/dict \
+            && mv zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json build-artifacts/zephyr/fprime-zephyr-deployment/dict
+          make submodules fprime-venv
+
       - name: Trigger Bootloader
         run: |
           make bootloader
           sleep 10
+
       - name: Copy Firmware
         run: |
           picotool load ./zephyr.uf2
           picotool reboot
+
       - name: Run Integration Tests
         run: |
           make test-integration

.pre-commit-config.yaml

@@ -33,3 +33,14 @@ repos:
     - id: ruff-check
       args: [--fix, --select, I] # import sorting
     - id: ruff-format
+
+- repo: https://github.com/econchick/interrogate/
+  rev: 1.7.0
+  hooks:
+    - id: interrogate
+      args:
+        - --ignore-init-method
+        - --omit-covered-files
+        - --fail-under=100
+        - -vv
+        - --color

FprimeZephyrReference/Components/Burnwire/Burnwire.cpp

@@ -72,13 +72,13 @@ void Burnwire ::schedIn_handler(FwIndexType portNum, U32 context) {
 // ----------------------------------------------------------------------
 
 void Burnwire ::START_BURNWIRE_cmdHandler(FwOpcodeType opCode, U32 cmdSeq) {
-    this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK);
     this->startBurn();
+    this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK);
 }
 
 void Burnwire ::STOP_BURNWIRE_cmdHandler(FwOpcodeType opCode, U32 cmdSeq) {
-    this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK);
     this->stopBurn();
+    this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK);
 }
 
 }  // namespace Components

FprimeZephyrReference/test/bootloader_trigger.py

@@ -1,21 +1,34 @@
+"""
+bootloader_trigger.py:
+
+This module is responsible for sending the command to trigger the bootloader mode
+on the PROVES hardware during integration tests. Once in bootloader mode, the device
+can accept firmware updates.
+"""
+
 import os
 import subprocess
 import time
 
 import pytest
 from fprime_gds.common.testing_fw.api import IntegrationTestAPI
+from int.common import cmdDispatch
 
 
 @pytest.fixture(scope="session", autouse=True)
 def start_gds(fprime_test_api_session: IntegrationTestAPI):
+    """Fixture to start GDS
+
+    GDS is used to send the bootloader commands.
+    """
     process = subprocess.Popen(["make", "gds-integration"], cwd=os.getcwd())
 
     gds_working = False
     timeout_time = time.time() + 30
     while time.time() < timeout_time:
         try:
             fprime_test_api_session.send_and_assert_command(
-                command="CdhCore.cmdDisp.CMD_NO_OP"
+                command=f"{cmdDispatch}.CMD_NO_OP"
             )
             gds_working = True
             break
@@ -28,6 +41,10 @@ def start_gds(fprime_test_api_session: IntegrationTestAPI):
 
 
 def test_bootloader(fprime_test_api: IntegrationTestAPI):
+    """Trigger bootloader mode on PROVES hardware"""
+    # Don't use proves_send_and_assert_command here because we don't expect
+    # a response from the bootloader trigger command. The device will reboot
+    # into bootloader mode and may not send a command completion event.
     fprime_test_api.send_command(
         "ReferenceDeployment.bootloaderTrigger.TRIGGER_BOOTLOADER"
     )

FprimeZephyrReference/test/int/burnwire_test.py

@@ -1,93 +1,64 @@
-# """
-# burnwire_test.py:
+"""
+burnwire_test.py:
 
-# Integration tests for the Burnwire component.
-# """
+Integration tests for the Burnwire component.
+"""
 
 import pytest
+from common import proves_send_and_assert_command
 from fprime_gds.common.testing_fw.api import IntegrationTestAPI
 
-# Constants
+burnwire = "ReferenceDeployment.burnwire"
 
 
 @pytest.fixture(autouse=True)
-def reset_burnwire(fprime_test_api: IntegrationTestAPI):
+def reset_burnwire(fprime_test_api: IntegrationTestAPI, start_gds):
     """Fixture to stop burnwire and clear histories before/after each test"""
     # Stop burnwire and clear before test
-    fprime_test_api.clear_histories()
     stop_burnwire(fprime_test_api)
     yield
     # Clear again after test to prevent residue
-    fprime_test_api.clear_histories()
     stop_burnwire(fprime_test_api)
 
 
 def stop_burnwire(fprime_test_api: IntegrationTestAPI):
-    fprime_test_api.send_and_assert_command(
-        "ReferenceDeployment.burnwire.STOP_BURNWIRE"
-    )
+    """Stop the burnwire and clear histories"""
+    proves_send_and_assert_command(fprime_test_api, f"{burnwire}.STOP_BURNWIRE")
 
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.SetBurnwireState", "OFF", timeout=10
-    )
+    fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "OFF", timeout=10)
 
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.BurnwireEndCount", timeout=2
-    )
-
-    received_events = fprime_test_api.get_event_subhistory()
-    print(f"Received events: {received_events}")
+    fprime_test_api.assert_event(f"{burnwire}.BurnwireEndCount", timeout=2)
 
 
 def test_01_start_and_stop_burnwire(fprime_test_api: IntegrationTestAPI, start_gds):
     """Test that burnwire starts and stops as expected"""
 
     # Start burnwire
-    fprime_test_api.send_and_assert_command(
-        "ReferenceDeployment.burnwire.START_BURNWIRE"
-    )
+    proves_send_and_assert_command(fprime_test_api, f"{burnwire}.START_BURNWIRE")
 
     # Wait for SetBurnwireState = ON
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.SetBurnwireState", "ON", timeout=2
-    )
+    fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "ON", timeout=2)
 
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.SafetyTimerState", timeout=2
-    )
+    fprime_test_api.assert_event(f"{burnwire}.SafetyTimerState", timeout=2)
 
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.SetBurnwireState", "OFF", timeout=10
-    )
+    fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "OFF", timeout=10)
 
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.BurnwireEndCount", timeout=2
-    )
+    fprime_test_api.assert_event(f"{burnwire}.BurnwireEndCount", timeout=2)
 
 
 def test_02_manual_stop_before_timeout(fprime_test_api: IntegrationTestAPI, start_gds):
     """Test that burnwire stops manually before the safety timer expires"""
 
     # Start burnwire
-    fprime_test_api.send_and_assert_command(
-        "ReferenceDeployment.burnwire.START_BURNWIRE"
-    )
+    proves_send_and_assert_command(fprime_test_api, f"{burnwire}.START_BURNWIRE")
 
     # Confirm Burnwire turned ON
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.SetBurnwireState", "ON", timeout=2
-    )
+    fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "ON", timeout=2)
 
     # # Stop burnwire before safety timer triggers
-    fprime_test_api.send_and_assert_command(
-        "ReferenceDeployment.burnwire.STOP_BURNWIRE"
-    )
+    proves_send_and_assert_command(fprime_test_api, f"{burnwire}.STOP_BURNWIRE")
 
     # Confirm Burnwire turned OFF
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.SetBurnwireState", "OFF", timeout=2
-    )
+    fprime_test_api.assert_event(f"{burnwire}.SetBurnwireState", "OFF", timeout=2)
 
-    fprime_test_api.assert_event(
-        "ReferenceDeployment.burnwire.BurnwireEndCount", timeout=2
-    )
+    fprime_test_api.assert_event(f"{burnwire}.BurnwireEndCount", timeout=2)

FprimeZephyrReference/test/int/common.py

@@ -0,0 +1,42 @@
+"""
+common.py:
+
+This module provides a functions and constants shared by
+integration tests for PROVES microcontroller hardware.
+"""
+
+from fprime_gds.common.testing_fw.api import IntegrationTestAPI
+from fprime_gds.common.testing_fw.predicates import event_predicate
+
+cmdDispatch = "CdhCore.cmdDisp"
+
+
+def proves_send_and_assert_command(
+    fprime_test_api: IntegrationTestAPI,
+    command: str,
+    args: list[str] = [],
+    events: list[event_predicate] = [],
+):
+    """Send command and assert completion
+
+    PROVES microcontroller hardware responds more slowly than typical FPrime
+    hardware which use microprocessors. As a result, some commands may
+    take longer to complete. This function clears histories before sending
+    the command, sets a longer timeout for command completion, and retries
+    up to 3 times if command assertion fails.
+    """
+    fprime_test_api.clear_histories()
+
+    for attempt in range(3):
+        try:
+            fprime_test_api.send_and_assert_command(
+                command,
+                args,
+                timeout=5,
+                max_delay=5,
+                events=events,
+            )
+            break
+        except AssertionError:
+            if attempt == 2:
+                raise