mosq__build.yml

name: "mosq: build-tests"

on:
  push:
    branches:
      - master
  pull_request:
    types: [opened, synchronize, reopened, labeled]

jobs:
  build_mosq:
    if: contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push'
    name: Mosquitto build
    strategy:
      matrix:
        idf_ver: ["latest", "release-v6.0", "release-v5.5", "release-v5.4", "release-v5.3", "release-v5.2", "release-v5.1"]
        example: ["broker"]
        include:
          # serverless_mqtt is not supported on >=v6.0 (esp-peer dependency)
          - idf_ver: "release-v5.3"
            example: "serverless_mqtt"
          - idf_ver: "release-v5.4"
            example: "serverless_mqtt"
          - idf_ver: "release-v5.5"
            example: "serverless_mqtt"

    runs-on: ubuntu-22.04
    container: espressif/idf:${{ matrix.idf_ver }}
    env:
      TEST_DIR: components/mosquitto/examples/${{ matrix.example }}
      TARGET_TEST: broker
      TARGET_TEST_DIR: build_esp32_default
    steps:
      - name: Checkout esp-protocols
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Build with IDF-${{ matrix.idf_ver }}
        shell: bash
        run: |
          . ${IDF_PATH}/export.sh
          pip install idf-component-manager idf-build-apps --upgrade
          python ci/build_apps.py -c ${TEST_DIR} -m components/mosquitto/.build-test-rules.yml
          if [ "${{ matrix.example }}" == "${TARGET_TEST}" ]; then
            # upload only the target test artifacts
            cd ${TEST_DIR}
            ${GITHUB_WORKSPACE}/ci/clean_build_artifacts.sh `pwd`/${TARGET_TEST_DIR}
            zip -qur artifacts.zip ${TARGET_TEST_DIR}
          fi
      - uses: actions/upload-artifact@v4
        if: ${{ matrix.example == 'broker' }}
        with:
          name: mosq_target_esp32_${{ matrix.idf_ver }}
          path: ${{ env.TEST_DIR }}/artifacts.zip
          if-no-files-found: error

  test_mosq:
    # Skip running on forks since it won't have access to secrets
    if: |
      github.repository == 'espressif/esp-protocols' &&
      ( contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push' )
    name: Mosquitto target test
    needs: build_mosq
    strategy:
      matrix:
        idf_ver: ["release-v5.4", "release-v5.5", "release-v6.0", "latest"]
    runs-on:
      - self-hosted
      - ESP32-ETHERNET-KIT
    env:
      TEST_DIR: components/mosquitto/examples/broker
      TARGET_TEST_DIR: build_esp32_default
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: mosq_target_esp32_${{ matrix.idf_ver }}
          path: ${{ env.TEST_DIR }}/ci/
      - name: Run Test
        working-directory: ${{ env.TEST_DIR }}
        run: |
          export PYENV_ROOT="$HOME/.pyenv"
          export PATH="$PYENV_ROOT/bin:$PATH"
          eval "$(pyenv init --path)"
          eval "$(pyenv init -)"
          if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then
            echo "Installing Python 3.12.6..."
            pyenv install -s 3.12.6
          fi
          if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then
            echo "Creating pyenv virtualenv 'myenv'..."
            pyenv virtualenv 3.12.6 myenv
          fi
          pyenv activate myenv
          python --version
          python -m pip install pytest-embedded-serial-esp pytest-embedded-idf pytest-rerunfailures pytest-timeout pytest-ignore-test-results
          unzip ci/artifacts.zip -d ci
          for dir in `ls -d ci/build_*`; do
          rm -rf build sdkconfig.defaults
          mv $dir build
          python -m pytest --log-cli-level DEBUG --junit-xml=./results_esp32_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=esp32
          done

  check_consistency:
    if: contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push'
    name: Checks that API docs and versions are consistent
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Checks API Docs and versions
        run: |
          sudo apt-get update
          sudo apt-get -y install doxygen
          pip install esp-doxybook
          cd components/mosquitto
          cp api.md api_orig.md
          ./generate_api_docs.sh
          diff -wB api.md api_orig.md
          # check version consistency
          CONFIG_VERSION=$(grep -Po '(?<=#define VERSION ")[^"]*' port/priv_include/config.h)
          CZ_VERSION=$(grep -Po '(?<=version: )[^"]*' .cz.yaml)
          COMP_VERSION=$(grep -Po '(?<=version: ")[^"]*' idf_component.yml)
          if [ "$CONFIG_VERSION" != "v$CZ_VERSION" ] || [ "$CONFIG_VERSION" != "v$COMP_VERSION" ]; then
            echo "Version mismatch detected:"
            echo "config.h: $CONFIG_VERSION"
            echo ".cz.yaml: $CZ_VERSION"
            echo "idf_component.yml: $COMP_VERSION"
            exit 1
          fi
          echo "Versions are consistent: $CONFIG_VERSION"

  build_idf_tests_with_mosq:
    if: |
      github.repository == 'espressif/esp-protocols' &&
      ( contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push' )
    name: Build IDF tests
    strategy:
      matrix:
        idf_ver: ["release-v5.5"]   # TODO: add release-v6.0/latest with esp-mqtt directly
        idf_target: ["esp32"]
        test: [ { app: publish, path: "tools/test_apps/protocols/mqtt/publish_connect_test" }]
    runs-on: ubuntu-22.04
    container: espressif/idf:${{ matrix.idf_ver }}
    env:
      TARGET_TEST_DIR: build_esp32_local_broker
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Build ${{ matrix.test.app }} with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}
        shell: bash
        run: |
          . ${IDF_PATH}/export.sh
          pip install idf-component-manager idf-build-apps --upgrade
          export MOSQUITTO_PATH=`pwd`/components/mosquitto
          # to use the actual version of mosquitto
          sed -i '/espressif\/mosquitto:/a \ \ \ \ override_path: "${MOSQUITTO_PATH}"' ${IDF_PATH}/${{matrix.test.path}}/main/idf_component.yml
          export PEDANTIC_FLAGS="-DIDF_CI_BUILD -Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function"
          export EXTRA_CFLAGS="${PEDANTIC_FLAGS} -Wstrict-prototypes"
          export EXTRA_CXXFLAGS="${PEDANTIC_FLAGS}"
          cd ${IDF_PATH}/${{matrix.test.path}}
          idf-build-apps find --config sdkconfig.ci.local_broker -vv --target ${{ matrix.idf_target }} --build-dir=${TARGET_TEST_DIR}
          idf-build-apps build --config sdkconfig.ci.local_broker -vv --target ${{ matrix.idf_target }} --build-dir=${TARGET_TEST_DIR}
          ${GITHUB_WORKSPACE}/ci/clean_build_artifacts.sh `pwd`/${TARGET_TEST_DIR}
          # to replace mqtt test configs with specific mosquitto markers
          python ${MOSQUITTO_PATH}/test/replace_decorators.py pytest_mqtt_publish_app.py ${TARGET_TEST_DIR}/pytest_mosquitto.py
          zip -qur ${GITHUB_WORKSPACE}/artifacts.zip ${TARGET_TEST_DIR}
      - uses: actions/upload-artifact@v4
        with:
          name: mosq_publish_esp32_${{ matrix.idf_ver }}
          path: artifacts.zip
          if-no-files-found: error

  test_idf_ci_with_mosq:
    # Skip running on forks since it won't have access to secrets
    if: |
      github.repository == 'espressif/esp-protocols' &&
      ( contains(github.event.pull_request.labels.*.name, 'mosquitto') || github.event_name == 'push' )
    name: Mosquitto IDF target tests
    needs: build_idf_tests_with_mosq
    strategy:
      matrix:
        idf_ver: ["release-v5.5"]
    runs-on:
      - self-hosted
      - ESP32-ETHERNET-KIT
    env:
      TEST_DIR: examples
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: mosq_publish_esp32_${{ matrix.idf_ver }}
          path: ${{ env.TEST_DIR }}/ci/
      - name: Run Test
        working-directory: ${{ env.TEST_DIR }}
        run: |
          export PYENV_ROOT="$HOME/.pyenv"
          export PATH="$PYENV_ROOT/bin:$PATH"
          eval "$(pyenv init --path)"
          eval "$(pyenv init -)"
          if ! pyenv versions --bare | grep -q '^3\.12\.6$'; then
            echo "Installing Python 3.12.6..."
            pyenv install -s 3.12.6
          fi
          if ! pyenv virtualenvs --bare | grep -q '^myenv$'; then
            echo "Creating pyenv virtualenv 'myenv'..."
            pyenv virtualenv 3.12.6 myenv
          fi
          pyenv activate myenv
          python --version
          python -m pip install pytest-embedded-serial-esp pytest-embedded-idf pytest-rerunfailures pytest-timeout pytest-ignore-test-results "paho-mqtt<2"  --upgrade
          unzip ci/artifacts.zip -d ci
          for dir in `ls -d ci/build_*`; do
          rm -rf build sdkconfig.defaults
          mv $dir build
          mv build/*.py .
          # Run only "test_mosquitto" marked tests
          python -m pytest --log-cli-level DEBUG --junit-xml=./results_esp32_${{ matrix.idf_ver }}_${dir#"ci/build_"}.xml --target=esp32 -m test_mosquitto
          done