feat(ci): Migrate macOS runners to Parallels VM

Author: petermESP

.gitlab/ci/build.yml

@@ -231,13 +231,18 @@ pytest_build_system_macos:
     - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats
     - .rules:build:macos
   tags:
-    - macos_shell
+    - macos
   parallel: 3
   variables:
     PYENV_VERSION: "3.9"
     # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions.
     CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache".
     CCACHE_MAXSIZE: "5G" # To preserve the limited Macbook storage. CCACHE automatically prunes old caches to fit the set limit.
+    # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path,
+    # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf.
+    # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
+    IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
+
 build_docker:
   extends:
     - .before_script:minimal

.gitlab/ci/common.yml

@@ -243,8 +243,6 @@ variables:
     # Ensure pyenv and PYENV_VERSION installed
     - eval "$(pyenv init -)"
     - *common-before_scripts
-    # On macOS, these tools need to be installed
-    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
     # remove idf-env.json, since it may contains enabled "features"
     - rm -f $IDF_TOOLS_PATH/idf-env.json
     # This adds tools (compilers) and the version-specific Python environment to PATH

.gitlab/ci/host-test.yml

@@ -348,7 +348,7 @@ test_pytest_macos:
     - .host_test_template
     - .before_script:build:macos
   tags:
-    - macos_shell
+    - macos
   artifacts:
     paths:
       - XUNIT_RESULT.xml
@@ -359,18 +359,22 @@ test_pytest_macos:
   variables:
     PYENV_VERSION: "3.9"
     PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
+    # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path,
+    # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf.
+    # GitLab sets the project dir to this template `<builds_dir>/<namespace>/<project_name>`
+    IDF_PATH: "/Users/espressif/builds/espressif/esp-idf"
   script:
     - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
       --target linux
       --pytest-apps
-      -m \"host_test and macos_shell\"
+      -m \"host_test and macos\"
       --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt"
       --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
     - python tools/ci/get_known_failure_cases_file.py
     - run_cmd pytest
       --target linux
-      -m \"host_test and macos_shell\"
+      -m \"host_test and macos\"
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
       --app-info-filepattern \"list_job_*.txt\"

examples/get-started/hello_world/pytest_hello_world.py

@@ -25,7 +25,7 @@ def test_hello_world_linux(dut: IdfDut) -> None:
 
 
 @pytest.mark.host_test
-@pytest.mark.macos_shell
+@pytest.mark.macos
 @idf_parametrize('target', ['linux'], indirect=['target'])
 def test_hello_world_macos(dut: IdfDut) -> None:
     dut.expect('Hello world!')

tools/ci/configure_ci_environment.sh

@@ -35,7 +35,7 @@ fi
 
 # https://ccache.dev/manual/latest.html#_configuring_ccache
 # Set ccache base directory to the project checkout path, to cancel out differences between runners
-export CCACHE_BASEDIR="${CI_PROJECT_DIR}"
+export CCACHE_BASEDIR="${IDF_PATH}"
 
 # host mapping volume to share ccache fbetween runner concurrent jobs
 export CCACHE_SLOPPINESS="time_macros"

tools/ci/idf_pytest/constants.py

@@ -63,7 +63,7 @@
 ENV_MARKERS = {
     # special markers
     'qemu': 'build and test using qemu, not real target',
-    'macos_shell': 'tests should be run on macos hosts',
+    'macos': 'tests should be run on macos hosts',
     # single-dut markers
     'generic': 'tests should be run on generic runners',
     'flash_suspend': 'support flash suspend feature',