ci: apply idf-ci for artifacts

Author: hfudev

.gitlab/ci/README.md

@@ -255,72 +255,4 @@ In ESP-IDF CI, there's a few more special rules are additionally supported to di
 
 ## Upload/Download Artifacts to Internal Minio Server
 
-### Users Without Access to Minio
-
-If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
-
-The log will look like this:
-
-```shell
-Pipeline ID    : 587355
-Job name       : build_clang_test_apps_esp32
-Job ID         : 40272275
-Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
-Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
-```
-
-### Users With Access to Minio
-
-#### Env Vars for Minio
-
-Minio takes these env vars to connect to the server:
-
-- `IDF_S3_SERVER`
-- `IDF_S3_ACCESS_KEY`
-- `IDF_S3_SECRET_KEY`
-- `IDF_S3_BUCKET`
-
-#### Artifacts Types and File Patterns
-
-The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
-
-#### Upload
-
-```shell
-python tools/ci/artifacts_handler.py upload
-```
-
- will upload the files that match the file patterns to minio object storage with name:
-
-`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
-
-For example, job 39043328 will upload these four files:
-
-- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/logs/build_pytest_examples_esp32/39043328.zip`
-- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
-
-#### Download
-
-You may run
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
-```
-
-to download all files of the pipeline, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
-```
-
-to download all files with the specified job name or pattern, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
-```
-
-to download all files with the specified job name or pattern and artifact type(s).
-
-You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`
+Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html)

.gitlab/ci/build.yml

@@ -21,7 +21,7 @@
     - pipeline_variables
   artifacts:
     paths:
-      # The other artifacts patterns are defined under tools/ci/artifacts_handler.py
+      # The other artifacts patterns are defined under .idf_ci.toml
       # Now we're uploading/downloading the binary files from our internal storage server
       #
       # keep the log file to help debug
@@ -44,7 +44,6 @@
       --modified-components ${MR_MODIFIED_COMPONENTS}
       --modified-files ${MR_MODIFIED_FILES}
       $TEST_BUILD_OPTS_EXTRA
-    - python tools/ci/artifacts_handler.py upload
 
 ######################
 # build_template_app #
@@ -206,7 +205,7 @@ build_clang_test_apps_esp32p4:
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - cd ${IDF_PATH}/tools/test_build_system
-    - python ${IDF_PATH}/tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest
       --cleanup-idf-copy
       --parallel-count ${CI_NODE_TOTAL:-1}

.gitlab/ci/common.yml

@@ -120,7 +120,7 @@ variables:
   source tools/ci/configure_ci_environment.sh
 
   # add extra python packages
-  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
 
 .setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
   # must use after setup_tools_except_target_test
@@ -217,7 +217,7 @@ variables:
 
 .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
   if [ $CI_JOB_STATUS = "failed" ]; then
-    python tools/ci/artifacts_handler.py upload --type logs
+    run_cmd idf-ci gitlab upload-artifacts --type log
   fi
 
 .before_script:minimal:

.gitlab/ci/host-test.yml

@@ -303,7 +303,7 @@ test_pytest_qemu:
       --only-test-related
       -m qemu
       --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target $IDF_TARGET
       --log-cli-level DEBUG
@@ -331,7 +331,7 @@ test_pytest_linux:
       --target linux
       --only-test-related
       --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target linux
       --embedded-services idf
@@ -365,7 +365,7 @@ test_pytest_macos:
       --only-test-related
       -m macos
       --modified-files ${MR_MODIFIED_FILES}
-    - python tools/ci/get_known_failure_cases_file.py
+    - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target linux
       -m macos

.gitlab/ci/pre_check.yml

@@ -144,9 +144,9 @@ pipeline_variables:
       fi
     - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
     - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
-    - idf-ci gitlab pipeline-variables >> pipeline.env
+    - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
     - cat pipeline.env
-    - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report
+    - run_cmd idf-ci gitlab upload-artifacts --type env
   artifacts:
     reports:
       dotenv: pipeline.env

.idf_ci.toml

@@ -9,6 +9,9 @@ IDF_CI_BUILD = "1"
 [gitlab]
 
 [gitlab.build_pipeline]
+workflow_name = "build_child_pipeline"
+presigned_json_job_name = 'generate_pytest_build_report'
+
 job_tags = ['build', 'shiny']
 job_template_name = '.dynamic_build_template'
 job_template_jinja = ''  # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
@@ -38,3 +41,57 @@ include:
   - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
   - tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml
 """
+
+[gitlab.artifacts.s3.debug]
+bucket = "idf-artifacts"
+patterns = [
+    '**/build*/bootloader/*.map',
+    '**/build*/bootloader/*.elf',
+    '**/build*/*.map',
+    '**/build*/*.elf',
+    # customized
+    '**/build*/esp_tee/*.map',
+    '**/build*/esp_tee/*.elf',
+    '**/build*/gdbinit/*',
+]
+
+[gitlab.artifacts.s3.flash]
+bucket = "idf-artifacts"
+patterns = [
+    '**/build*/bootloader/*.bin',
+    '**/build*/*.bin',
+    '**/build*/partition_table/*.bin',
+    '**/build*/flasher_args.json',
+    '**/build*/flash_project_args',
+    '**/build*/config/sdkconfig.json',
+    '**/build*/sdkconfig',
+    '**/build*/project_description.json',
+    # customized
+    '**/build*/esp_tee/*.bin',
+]
+
+[gitlab.artifacts.s3.log]
+bucket = "idf-artifacts"
+patterns = [
+    '**/build*/build_log.txt',
+    '**/build*/size.json',
+]
+
+[gitlab.artifacts.s3.junit]
+bucket = "idf-artifacts"
+patterns = [
+    'XUNIT_RESULT_*.xml',
+]
+
+[gitlab.artifacts.s3.env]
+bucket = "idf-artifacts"
+patterns = [
+    'pipeline.env',
+]
+
+[gitlab.artifacts.s3.longterm]
+bucket = "longterm"
+if_clause = '"$CI_COMMIT_REF_NAME" == "master"'
+patterns = [
+    '**/build*/size.json',
+]