Add: smoke tests workflow (#1215)

Co-authored-by: KarolinaPomian <[email protected]>
Co-authored-by: KarolinaPomian <[email protected]>

Author: staszczuk

.github/workflows/smoke-tests.yml

@@ -0,0 +1,205 @@
+name: smoke-tests-bare-metal
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - 'maint-**'
+  pull_request:
+    branches:
+      - main
+      - 'maint-**'
+env:
+  BUILD_TYPE: 'Release'
+  DPDK_VERSION: '25.03'
+  DPDK_REBUILD: 'false'
+permissions:
+  contents: read
+jobs:
+  validation-build-mtl:
+    runs-on: [Linux, self-hosted, DPDK]
+    timeout-minutes: 60
+    outputs:
+      pipenv-activate: ${{ steps.pipenv-install.outputs.VIRTUAL_ENV }}
+    steps:
+      - name: 'preparation: Harden Runner'
+        uses: step-security/harden-runner@6c439dc8bdf85cadbbce9ed30d1c7b959517bc49 # v2.12.2
+        with:
+          egress-policy: audit
+      - name: 'preparation: Restore valid repository owner and print env'
+        if: always()
+        run: |
+          sudo chown -R "${USER}" "$(pwd)" || true
+          env | grep TEST_ || true
+      - name: 'preparation: Checkout MTL'
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: 'pstaszczuk/smoke-tests'
+      - name: 'preparation: Checkout DPDK'
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        if: env.DPDK_REBUILD == 'true'
+        with:
+          repository: 'DPDK/dpdk'
+          ref: 'v${{  env.DPDK_VERSION  }}'
+          path: 'dpdk'
+      - name: 'preparation: Checkout openh264'
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: 'cisco/openh264'
+          ref: 'openh264v2.4.0'
+          path: 'openh264'
+      - name: 'preparation: Checkout FFmpeg'
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          repository: 'FFmpeg/FFmpeg'
+          ref: 'release/7.0'
+          path: 'ffmpeg'
+      - name: 'configuration: Install the build dependency'
+        run: |
+          sudo apt update
+          sudo apt-get remove -y pipenv || true
+          sudo apt-get install -y \
+              git gcc meson tar zip \
+              pkg-config \
+              python3 \
+              python3-pyelftools \
+              python3-virtualenv \
+              python3-pip \
+              libnuma-dev \
+              libjson-c-dev \
+              libpcap-dev \
+              libgtest-dev \
+              libsdl2-dev \
+              libsdl2-ttf-dev \
+              libssl-dev \
+              systemtap-sdt-dev \
+              libbpf-dev \
+              libelf1 \
+              gstreamer1.0-plugins-base \
+              gstreamer1.0-plugins-good \
+              gstreamer1.0-tools \
+              gstreamer1.0-libav \
+              libgstreamer1.0-dev \
+              libgstreamer-plugins-base1.0-dev
+      - name: 'configuration: Apply dpdk patches'
+        if: env.DPDK_REBUILD == 'true'
+        run: |
+          patch -d "dpdk" -p1 -i <(cat patches/dpdk/${{  env.DPDK_VERSION  }}/*.patch)
+      - name: 'installation: Build dpdk'
+        working-directory: dpdk
+        if: env.DPDK_REBUILD == 'true'
+        run: |
+          meson build
+          ninja -C build
+          sudo ninja -C build install
+      - name: 'installation: Build mtl'
+        run: |
+          ./build.sh
+          sudo ldconfig
+      - name: 'installation: Build openh264'
+        working-directory: openh264
+        run: |
+          make -j "$(nproc)"
+          sudo make install
+          sudo ldconfig
+      - name: 'installation: Build FFmpeg'
+        working-directory: ffmpeg
+        run: |
+          git am ../ecosystem/ffmpeg_plugin/7.0/*.patch
+          cp ../ecosystem/ffmpeg_plugin/mtl_*.c -rf libavdevice/
+          cp ../ecosystem/ffmpeg_plugin/mtl_*.h -rf libavdevice/
+          ./configure --enable-shared --disable-static --enable-nonfree --enable-pic --enable-gpl --enable-libopenh264 --enable-encoder=libopenh264 --enable-mtl
+          make -j "$(nproc)"
+          sudo make install
+          sudo ldconfig
+      - name: 'installation: Build GStreamer'
+        working-directory: ecosystem/gstreamer_plugin
+        run: |
+          ./build.sh
+      - name: 'installation: Install pipenv environment'
+        working-directory: tests/validation
+        id: pipenv-install
+        run: |
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          echo "VIRTUAL_ENV=$PWD/venv/bin/activate" >> "$GITHUB_ENV"
+  validation-run-tests:
+    needs: [validation-build-mtl]
+    runs-on: [Linux, self-hosted, DPDK]
+    timeout-minutes: 720
+    env:
+      PYTEST_RETRIES: '3'
+    steps:
+      - name: 'preparation: Harden Runner'
+        uses: step-security/harden-runner@6c439dc8bdf85cadbbce9ed30d1c7b959517bc49 # v2.12.2
+        with:
+          egress-policy: audit
+      - name: 'preparation: Evaluate choosen validation-test-port-p and validation-test-port-r'
+        run: |
+          eval "export TEST_PORT_P=TEST_VF_PORT_P_0"
+          eval "export TEST_PORT_R=TEST_VF_PORT_P_1"
+          echo "TEST_PORT_P=${TEST_PORT_P}" >> "$GITHUB_ENV"
+          echo "TEST_PORT_R=${TEST_PORT_R}" >> "$GITHUB_ENV"
+          echo "TEST_PORT_P=${TEST_PORT_P}"
+          echo "TEST_PORT_R=${TEST_PORT_R}"
+      - name: 'preparation: Kill MtlManager and pytest routines'
+        run: |
+          sudo killall -SIGINT pipenv || true
+          sudo killall -SIGINT pytest || true
+          sudo killall -SIGINT MtlManager || true
+      - name: 'preparation: Create VFs'
+        run: |
+          sudo rmmod irdma || true
+          sudo ./script/nicctl.sh create_vf "${TEST_PF_PORT_P}" || true
+          sudo ./script/nicctl.sh create_vf "${TEST_PF_PORT_R}" || true
+      - name: 'preparation: Start MtlManager at background'
+        run: |
+          sudo MtlManager &
+      - name: 'execution: Run validation-bare-metal tests in virtual environment'
+        run: |
+          sudo tests/validation/venv/bin/python3 -m pytest --topology_config=tests/validation/configs/topology_config.yaml --test_config=tests/validation/configs/test_config.yaml -m smoke  --template=html/index.html --report=report.html
+      - name: "upload report"
+        id: upload-report
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: smoke-tests-report
+          path: |
+            report.html
+      - name: "Add report to summary"
+        if: always()
+        run: |
+          {
+            echo "## Smoke Tests Report"
+            echo ""
+            # Check if JSON report exists
+            if [ -f "report.json" ]; then
+              # Parse JSON report
+                PASSED=$(jq '.summary.passed // 0' report.json)
+                FAILED=$(jq '.summary.failed // 0' report.json)
+                SKIPPED=$(jq '.summary.skipped // 0' report.json)
+                ERROR=$(jq '.summary.errors // 0' report.json)
+              # Add summary stats
+              echo "| Status | Count |"
+              echo "| ------ | ----- |"
+              echo "| ✅ Passed | ${PASSED:-0} |"
+              echo "| ❌ Failed | ${FAILED:-0} |"
+              echo "| ⚠️ Error | ${ERROR:-0} |"
+              echo "| ⏭️ Skipped | ${SKIPPED:-0} |"
+              echo ""
+              # Add test result details if available
+              TOTAL=$((${PASSED:-0} + ${FAILED:-0} + ${ERROR:-0} + ${SKIPPED:-0}))
+              echo "**Total Tests:** $TOTAL"
+              echo ""
+              if [ "${FAILED:-0}" -gt 0 ] || [ "${ERROR:-0}" -gt 0 ]; then
+                echo "❌ **Some tests failed!** Please check the detailed report."
+              else
+                echo "✅ **All tests passed!**"
+              fi
+              echo ""
+              # Add link to full report artifact
+              echo "📄 [Download Full HTML Report](https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/artifacts/${{ steps.upload-report.outputs.artifact-id }})"
+            else
+              echo "❌ No report.json file was generated"
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"

tests/validation/configs/test_config.yaml

@@ -1,14 +1,14 @@
-build: /path/to/Media-Transport-Library
-mtl_path: /path/to/Media-Transport-Library
-media_path: /path/to/media
+build: .
+mtl_path: .
+media_path: /mnt/media
 capture_cfg:
   enable: false
   test_name: test_name
-  pcap_dir: /path/to/pcap_files
+  pcap_dir: /home/ubuntu/pcap_files
   capture_time: 5
   interface: null
 ramdisk:
-  pcap_dir: /path/to/pcap_files
+  pcap_dir: /home/ubuntu/pcap_files
   tmpfs_size: 768G
   tmpfs_name: tmpfs_name
   use_sudo: true

tests/validation/configs/topology_config.yaml

@@ -9,9 +9,5 @@ hosts:
       - pci_device: 8086:1592
         interface_index: 0 # all
     connections:
-      - ip_address: ip_address
-        connection_type: SSHConnection
-        connection_options:
-          port: 22
-          username: username
-          password: passowrd
+      - ip_address: 127.0.0.1
+        connection_type: LocalConnection

tests/validation/conftest.py

@@ -65,7 +65,7 @@ def mtl_path(test_config: dict) -> str:
 
 @pytest.fixture(scope="session", autouse=True)
 def keep(request):
-    keep = request.config.getoption("--keep")
+    keep = request.config.getoption("--keep", None)
     if keep is None:
         keep = "none"
     if keep.lower() not in ["all", "failed", "none"]:
@@ -76,7 +76,7 @@ def keep(request):
 
 @pytest.fixture(scope="session", autouse=True)
 def dmesg(request):
-    dmesg = request.config.getoption("--dmesg")
+    dmesg = request.config.getoption("--dmesg", None)
     if dmesg is None:
         dmesg = "keep"
     if dmesg.lower() not in ["clear", "keep"]:
@@ -182,7 +182,10 @@ def log_session():
     os.makedirs(path, exist_ok=True)
     os.symlink(folder, path_symlink)
     yield
-    shutil.copy("pytest.log", f"{LOG_FOLDER}/latest/pytest.log")
+    if os.path.exists("pytest.log"):
+        shutil.copy("pytest.log", f"{LOG_FOLDER}/latest/pytest.log")
+    else:
+        logging.warning("pytest.log not found, skipping copy")
     csv_write_report(f"{LOG_FOLDER}/latest/report.csv")
 
 

tests/validation/mtl_engine/RxTxApp.py

@@ -10,6 +10,7 @@
 import time
 
 from create_pcap_file.tcpdump import TcpDumpRecorder
+from mfd_connect import SSHConnection
 
 from . import rxtxapp_config
 from .execute import log_fail, run
@@ -523,8 +524,9 @@ def execute_test(
     remote_conn = remote_host.connection
     config_file = f"{build}/tests/config.json"
     f = remote_conn.path(config_file)
-    json_content = config_json.replace('"', '\\"')
-    f.write_text(json_content)
+    if isinstance(remote_conn, SSHConnection):
+        config_json = config_json.replace('"', '\\"')
+    f.write_text(config_json, encoding="utf-8")
     config_path = os.path.join(build, config_file)
     log_to_file(f"Config file written to remote host: {config_path}", host, build)
 
@@ -743,8 +745,9 @@ def execute_perf_test(
     remote_conn = host.connection
     config_file = f"{build}/tests/config.json"
     f = remote_conn.path(config_file)
-    json_content = config_json.replace('"', '\\"')
-    f.write_text(json_content)
+    if isinstance(remote_conn, SSHConnection):
+        config_json = config_json.replace('"', '\\"')
+    f.write_text(config_json, encoding="utf-8")
     config_path = os.path.join(build, config_file)
     log_to_file(
         f"Performance config file written to remote host: {config_path}", host, build
@@ -1679,13 +1682,11 @@ def execute_dual_test(
 
     # Prepare TX config
     tx_f = tx_host.connection.path(build, "tests", "tx_config.json")
-    tx_json_content = tx_config.replace('"', '\\"')
-    tx_f.write_text(tx_json_content)
+    tx_f.write_text(tx_config, encoding="utf-8")
 
     # Prepare RX config
-    rx_f = tx_host.connection.path(build, "tests", "tx_config.json")
-    rx_json_content = rx_config.replace('"', '\\"')
-    rx_f.write_text(rx_json_content)
+    rx_f = tx_host.connection.path(build, "tests", "rx_config.json")
+    rx_f.write_text(rx_config, encoding="utf-8")
 
     # Adjust test_time for high-res/fps/replicas
     if (

tests/validation/mtl_engine/ffmpeg_app.py

@@ -8,6 +8,7 @@
 import re
 import time
 
+from mfd_connect import SSHConnection
 from mtl_engine.RxTxApp import prepare_tcpdump
 
 from . import rxtxapp_config
@@ -78,9 +79,9 @@ def log_to_file(message: str, host, build: str):
 
     if f.exists():
         current_content = f.read_text()
-        f.write_text(current_content + log_entry)
+        f.write_text(current_content + log_entry, encoding="utf-8")
     else:
-        f.write_text(log_entry)
+        f.write_text(log_entry, encoding="utf-8")
 
 
 def execute_test(
@@ -821,8 +822,9 @@ def generate_rxtxapp_rx_config(
         config_json = json.dumps(config, indent=4)
         remote_conn = host.connection
         f = remote_conn.path(config_file)
-        json_content = config_json.replace('"', '\\"')
-        f.write_text(json_content)
+        if isinstance(remote_conn, SSHConnection):
+            config_json = config_json.replace('"', '\\"')
+        f.write_text(config_json, encoding="utf-8")
 
         logger.info("Config file written successfully")
         log_to_file(f"Generated RX config file: {config_file}", host, build)
@@ -887,8 +889,9 @@ def generate_rxtxapp_rx_config_multiple(
         config_json = json.dumps(config, indent=4)
         remote_conn = host.connection
         f = remote_conn.path(config_file)
-        json_content = config_json.replace('"', '\\"')
-        f.write_text(json_content)
+        if isinstance(remote_conn, SSHConnection):
+            config_json = config_json.replace('"', '\\"')
+        f.write_text(config_json, encoding="utf-8")
 
         logger.info("Multiple config file written successfully")
         log_to_file(f"Generated RX multiple config file: {config_file}", host, build)
@@ -950,8 +953,9 @@ def generate_rxtxapp_tx_config(
         config_json = json.dumps(config, indent=4)
         remote_conn = host.connection
         f = remote_conn.path(config_file)
-        json_content = config_json.replace('"', '\\"')
-        f.write_text(json_content)
+        if isinstance(remote_conn, SSHConnection):
+            config_json = config_json.replace('"', '\\"')
+        f.write_text(config_json, encoding="utf-8")
 
         logger.info("TX Config file written successfully")
         log_to_file(f"Generated TX config file: {config_file}", host, build)

tests/validation/pytest.ini

@@ -1,2 +1,4 @@
 [pytest]
-log_file = pytest.log
+log_file=./tests/validation/pytest.log
+markers=
+    smoke: mark test as part of the smoke test suite

tests/validation/tests/single/ffmpeg/test_rx_ffmpeg_tx_ffmpeg.py

@@ -13,7 +13,7 @@
     [
         ("i1080p25", 2),
         ("i1080p50", 2),
-        ("i1080p60", 4),
+        pytest.param("i1080p60", 4, marks=pytest.mark.smoke),
         ("i2160p60", 6),
     ],
 )