Merge branch 'feat/split_pytest_requirements' into 'master'

feat: split pytest requirements to base and test-specific

Closes IDFCI-2307 and IDFCI-2376

See merge request espressif/esp-idf!33429

Author: hfudev

.gitlab/ci/common.yml

@@ -147,16 +147,16 @@ variables:
   # install latest python packages
   # target test jobs
   if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-pytest
+    run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
   elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
     run_cmd bash install.sh --enable-ci --enable-docs
   elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-pytest
+    run_cmd bash install.sh --enable-ci
   else
     if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
       run_cmd bash install.sh --enable-ci
     else
-      run_cmd bash install.sh --enable-ci --enable-pytest
+      run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
     fi
   fi
 
@@ -223,6 +223,8 @@ variables:
     - *common-before_scripts
     # On macOS, these tools need to be installed
     - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
+    # remove idf-env.json, since it may contains enabled "features"
+    - rm -f $IDF_TOOLS_PATH/idf-env.json
     - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
     - *setup_tools_and_idf_python_venv

components/spi_flash/esp32s3/Kconfig.flash_freq

@@ -10,10 +10,11 @@ choice ESPTOOLPY_FLASHFREQ
             - Flash 120 MHz SDR mode is stable.
             - Flash 120 MHz DDR mode is an experimental feature, it works when
               the temperature is stable.
-                Risks:
-                    If your chip powers on at a certain temperature, then after the temperature
-                    increases or decreases by approximately 20 Celsius degrees (depending on the
-                    chip), the program will crash randomly.
+
+              Risks:
+              If your chip powers on at a certain temperature, then after the temperature
+              increases or decreases by approximately 20 Celsius degrees (depending on the
+              chip), the program will crash randomly.
     config ESPTOOLPY_FLASHFREQ_80M
         bool "80 MHz"
     config ESPTOOLPY_FLASHFREQ_40M

docs/en/contribute/esp-idf-tests-with-pytest.rst

@@ -33,12 +33,18 @@ On the host side, ESP-IDF employs the pytest framework (alongside certain pytest
 Installation
 ============
 
-All dependencies could be installed by running the ESP-IDF install script with the ``--enable-pytest`` argument:
+All basic dependencies could be installed by running the ESP-IDF install script with the ``--enable-pytest`` argument:
 
 .. code-block:: bash
 
     $ install.sh --enable-pytest
 
+Additional test script specific dependencies could be installed separately by running the ESP-IDF install script with the ``--enable-pytest-specific`` argument:
+
+.. code-block:: bash
+
+    $ install.sh --enable-test-specific
+
 Several mechanisms have been implemented to ensure the successful execution of the installation processes. If you encounter any issues during installation, please submit an issue report to our `GitHub issue tracker <https://github.com/espressif/esp-idf/issues>`__.
 
 Common Concepts

docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst

@@ -33,12 +33,18 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自
 安装
 ============
 
-所有依赖项都可以通过执行 ESP-IDF 安装脚本 ``--enable-pytest`` 进行安装：
+基础依赖项可以通过执行 ESP-IDF 安装脚本 ``--enable-pytest`` 进行安装：
 
 .. code-block:: bash
 
     $ install.sh --enable-pytest
 
+额外的测试脚本依赖项可以通过执行 ESP-IDF 安装脚本 ``--enable-pytest-specific`` 进行安装：
+
+.. code-block:: bash
+
+    $ install.sh --enable-test-specific
+
 上面的脚本已预先实现了一些机制，以确保所有安装过程顺利进行。如果您在安装过程中遇到任何问题，请在 `GitHub Issue 版块 <https://github.com/espressif/esp-idf/issues>`__ 上提交问题说明。
 
 常见概念

tools/ci/check_requirement_files.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 # SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import json
 import os
 import re
@@ -13,7 +12,7 @@
 JSON_PATH = os.path.join(IDF_PATH, 'tools', 'requirements.json')
 SCHEMA_PATH = os.path.join(IDF_PATH, 'tools', 'requirements_schema.json')
 REQ_DIR = os.path.join(IDF_PATH, 'tools', 'requirements')
-RE_FEATURE = re.compile(r'requirements\.(\w+)\.txt')
+RE_FEATURE = re.compile(r'requirements\.([\w-]+)\.txt')
 
 
 def action_validate(req_obj: Any) -> None:  # "Any" because we are checking this in this script

tools/requirements.json

@@ -19,6 +19,12 @@
             "optional": true,
             "requirement_path": "tools/requirements/requirements.pytest.txt"
         },
+        {
+            "name": "test-specific",
+            "description": "Packages for specific test scripts",
+            "optional": true,
+            "requirement_path": "tools/requirements/requirements.test-specific.txt"
+        },
         {
             "name": "ci",
             "description": "Packages for ESP-IDF CI scripts",

tools/requirements/requirements.pytest.txt

@@ -15,21 +15,3 @@ minio
 # build
 python-gitlab
 idf-build-apps
-
-# dependencies in pytest test scripts
-scapy
-websocket-client
-netifaces
-rangehttpserver
-dbus-python; sys_platform == 'linux'
-protobuf
-bleak
-paho-mqtt
-paramiko
-netmiko
-
-# iperf_test_util
-pyecharts
-
-# for twai tests, communicate with socket can device (e.g. Canable)
-python-can

tools/requirements/requirements.test-specific.txt

@@ -0,0 +1,20 @@
+# Python package requirements for specific test scripts in ESP-IDF.
+# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-test-specific"
+
+# dependencies in pytest test scripts
+scapy
+websocket-client
+netifaces
+rangehttpserver
+dbus-python; sys_platform == 'linux'
+protobuf
+bleak
+paho-mqtt
+paramiko
+netmiko
+
+# iperf_test_util
+pyecharts
+
+# for twai tests, communicate with socket can device (e.g. Canable)
+python-can

tools/test_apps/system/.build-test-rules.yml

@@ -45,6 +45,9 @@ tools/test_apps/system/gdb_loadable_elf:
     - if: IDF_TARGET == "esp32c6" or IDF_TARGET == "esp32h2"
       temporary: true
       reason: target esp32c6, esp32h2 is not supported yet
+    - if: IDF_TARGET == "esp32p4"
+      temporary: true
+      reason: build failed. IDF-11133
 
 tools/test_apps/system/longjmp_test:
   enable:

tools/test_apps/system/gdb_loadable_elf/README.md

@@ -1,5 +1,5 @@
-| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C61 | ESP32-P4 | ESP32-S2 | ESP32-S3 |
-| ----------------- | ----- | -------- | -------- | -------- | --------- | -------- | -------- | -------- |
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C61 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | --------- | -------- | -------- |
 
 # Loadable ELF test application