Merge pull request #363 from david-cermak/feat/modem_test_ota

feat(modem): Add OTA test to exercise modem layers

Author: david-cermak

.github/workflows/modem__build-host-tests.yml

@@ -8,25 +8,19 @@ on:
     types: [opened, synchronize, reopened, labeled]
 
 jobs:
-  build_esp_modem:
+  build_esp_modem_examples:
     if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push'
-    name: Build
+    name: Build examples
     strategy:
       matrix:
-        idf_ver: ["latest", "release-v4.2", "release-v4.3", "release-v4.4", "release-v5.0"]
+        idf_ver: ["latest", "release-v4.3", "release-v4.4", "release-v5.0"]
         example: ["pppos_client", "modem_console", "modem_tcp_client", "ap_to_pppos", "simple_cmux_client"]
         exclude:
-          - idf_ver: "release-v4.2"
-            example: simple_cmux_client
-          - idf_ver: "release-v4.2"
-            example: modem_tcp_client
           - idf_ver: "release-v4.3"
             example: modem_tcp_client
           - idf_ver: "release-v4.4"
             example: modem_tcp_client
         include:
-          - idf_ver: "release-v4.2"
-            skip_config: usb
           - idf_ver: "release-v4.3"
             skip_config: usb
           - idf_ver: "release-v5.0"
@@ -50,7 +44,33 @@ jobs:
           . ${IDF_PATH}/export.sh
           python -m pip install idf-build-apps
           cd $GITHUB_WORKSPACE/protocols
-          python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }} -m components/esp_modem/examples/.build-test-rules.yml
+          python ./ci/build_apps.py components/esp_modem/examples/${{ matrix.example }} -m components/esp_modem/.build-test-rules.yml
+
+  build_esp_modem_tests:
+    if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push'
+    name: Build tests
+    strategy:
+      matrix:
+        idf_ver: ["release-v5.0", "release-v5.1", "latest"]
+        test: ["target", "target_ota"]
+
+    runs-on: ubuntu-20.04
+    container: espressif/idf:${{ matrix.idf_ver }}
+    steps:
+      - name: Checkout esp-protocols
+        uses: actions/checkout@v3
+        with:
+          path: protocols
+      - name: Build ${{ matrix.test }} with IDF-${{ matrix.idf_ver }}
+        env:
+          EXPECTED_WARNING: ${{ matrix.warning }}
+        shell: bash
+        run: |
+          . ${IDF_PATH}/export.sh
+          python -m pip install idf-build-apps
+          cd $GITHUB_WORKSPACE/protocols
+          python ./ci/build_apps.py components/esp_modem/test/${{ matrix.test }} -m components/esp_modem/.build-test-rules.yml
+
 
   host_test_esp_modem:
     if: contains(github.event.pull_request.labels.*.name, 'modem') || github.event_name == 'push'

components/esp_modem/examples/.build-test-rules.yml renamed to components/esp_modem/.build-test-rules.yml

@@ -29,6 +29,7 @@ class Tls {
     int read(unsigned char *buf, size_t len);
     [[nodiscard]] bool set_own_cert(const_buf crt, const_buf key);
     [[nodiscard]] bool set_ca_cert(const_buf crt);
+    bool set_hostname(const char *name);
     virtual int send(const unsigned char *buf, size_t len) = 0;
     virtual int recv(unsigned char *buf, size_t len) = 0;
     size_t get_available_bytes();

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/include/mbedtls_wrap.hpp

@@ -116,6 +116,16 @@ bool Tls::set_ca_cert(const_buf crt)
     return true;
 }
 
+bool Tls::set_hostname(const char *name)
+{
+    int ret = mbedtls_ssl_set_hostname(&ssl_, name);
+    if (ret < 0) {
+        print_error("mbedtls_ssl_set_hostname", ret);
+        return false;
+    }
+    return true;
+}
+
 Tls::Tls()
 {
     mbedtls_x509_crt_init(&public_cert_);

components/esp_modem/examples/modem_tcp_client/components/extra_tcp_transports/mbedtls_wrap.cpp

@@ -31,7 +31,7 @@
  */
 struct esp_modem_vfs_uart_creator {
     const char *dev_name;                       /*!< VFS device name, e.g. /dev/uart/n */
-    const struct esp_modem_uart_term_config uart;     /*!< UART driver init struct */
+    struct esp_modem_uart_term_config uart;     /*!< UART driver init struct */
 };
 
 /**

components/esp_modem/include/vfs_resource/vfs_create.hpp

@@ -0,0 +1,8 @@
+# The following lines of boilerplate have to be in your project's CMakeLists
+# in this exact order for cmake to work correctly
+cmake_minimum_required(VERSION 3.8)
+
+set(EXTRA_COMPONENT_DIRS "../.." "../../examples/modem_tcp_client/components")
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(ota_test)

components/esp_modem/test/target_ota/CMakeLists.txt

@@ -0,0 +1,27 @@
+# Target test running OTA update
+
+## Overview
+
+The aim of this test is to exercise the most commonly failing scenario, running OTA over PPPoS with https.
+
+This project opens a data session, runs basic mqtt operations and initiates OTA update.
+It supports the following test configurations:
+* Using a real modem device (default config)
+* Using VFS device (only to exercise VFS DTE)
+* Using network-only DCE (connecting directly to PPP server) -- needs some configuration
+
+### Configuring the PPP server
+
+You need to run these applications on your host machine:
+* PPP server
+```bash
+sudo pppd /dev/ttyUSB1 115200 192.168.11.1:192.168.11.2 ms-dns 8.8.8.8 modem local noauth debug nocrtscts nodetach +ipv6
+```
+* MQTT broker: Running mosquitto in the default config is enough, configuring the broker's URL to the local PPP address: `config.broker.address.uri = "mqtt://192.168.11.1";`
+* HTTP server: Need to support HTTP/1.1 (to support ranges). You can use the script `http_server.py` and configure the OTA endpoint as `"https://192.168.11.1:1234/esp32.bin"`
+
+## Potential issues
+
+When running this test it is expected to experience some buffer overflows or connection interruption.
+The modem library should recover from these failure modes and continue and complete OTA update.
+These issues are expected, since UART ISR is deliberately not placed into IRAM in the test configuration to exhibit some minor communication glitches.

components/esp_modem/test/target_ota/README.md

@@ -0,0 +1,3 @@
+idf_component_register(SRCS manual_ota.cpp transport_batch_tls.cpp
+                       INCLUDE_DIRS "."
+                       PRIV_REQUIRES extra_tcp_transports esp_http_client app_update)