From 5d310c01e25fff4bb9f5eaecaf0d77b5b315ee28 Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 2 May 2025 09:46:29 +0100 Subject: [PATCH 1/9] Added coverage-upload-options & codecov.yml --- .github/codecov.yml | 21 +++++++++++++++++++++ .yamato/upm-ci.yml | 19 +------------------ 2 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 .github/codecov.yml diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 0000000000..a3d69e87f8 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,21 @@ +codecov: + max_report_age: false + require_ci_to_pass: false + notify: + wait_for_ci: false + +coverage: + precision: 2 + round: down + range: "50...70" + status: + project: false + patch: false + default_rules: + flag_coverage_not_uploaded_behavior: exclude + +comment: false + +flag_management: + default_rules: + carryforward: true diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 36ecc3e0e4..1f957cbf17 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -27,24 +27,7 @@ # Run upm-ci verification tests as well as tests contained in the package. - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION% {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" - --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package" {% endif %} - {% if platform.installscript %} - - {{ unity_downloader_install }} - - {{ platform.installscript }} %EDITOR_VERSION% - {% endif %} - # ADBv2 on 2019.4 causes the test runner to not start on initial import when the - # samples are in the package. Move the samples back into the project. - - move /Y .\Packages\com.unity.inputsystem\Samples .\Assets - - move /Y .\Packages\com.unity.inputsystem\Samples.meta .\Assets - # Now run our full test suite that sits in Assets/Tests by running UTR on our project. - - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if platform.name == "win" and category.name == "functional" %}--enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project {% endif %} - # Merge all the code coverage results into one report. - {% if platform.name == "win" and category.name == "functional" %} - - '{{ yamato_source_dir_win }}/.Editor/Unity.exe -projectPath {{ yamato_source_dir_win }} -batchmode -debugCodeOptimization -enableCodeCoverage -coverageResultsPath {{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/ -coverageOptions "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;sourcePaths:{{ yamato_source_dir_win }}/Packages/com.unity.inputsystem/" -quit' - # Delete the Package and Project reports, only keep the final merged one to keep Aritfacts.zip smaller - - rmdir /s/q "{{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package" - - rmdir /s/q "{{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project" - {% endif %} + --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.test.unity3d.com;git-service:github;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} after: - {{ instabilities_install_win }} - {{ instabilities_run_win }} From e08cd2c0acceaaa255eed8f824ccdc806ad542aa Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Wed, 14 May 2025 16:53:50 +0100 Subject: [PATCH 2/9] Testing with new package ci (draft) image --- .yamato/config.metadata | 2 +- .yamato/upm-ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/config.metadata b/.yamato/config.metadata index 565df3b2ac..602c324d41 100644 --- a/.yamato/config.metadata +++ b/.yamato/config.metadata @@ -19,7 +19,7 @@ upm_ci_install: npm install -g upm-ci-utils@stable --registry https://artifactor platforms_win: - name: win type: Unity::VM - image: package-ci/win10:v4 + image: pets-ops/draft:M97c169bdfd16-4006556-1 flavor: b1.large - name: win_standalone type: Unity::VM diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 1f957cbf17..7ec042838f 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -27,7 +27,7 @@ # Run upm-ci verification tests as well as tests contained in the package. - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION% {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" - --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.test.unity3d.com;git-service:github;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} + --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.test.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} after: - {{ instabilities_install_win }} - {{ instabilities_run_win }} From 0e1265832f728a18687482339ced334ce729ab40 Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 16 May 2025 10:16:24 +0100 Subject: [PATCH 3/9] Testing upload to Codecov.stg --- .yamato/upm-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 7ec042838f..f5e25057f8 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -27,7 +27,7 @@ # Run upm-ci verification tests as well as tests contained in the package. - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION% {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" - --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.test.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} + --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.stg.unity3d.com;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} after: - {{ instabilities_install_win }} - {{ instabilities_run_win }} From 91c26117226a50867fb12926a73e8d82bbbb343b Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 16 May 2025 10:18:36 +0100 Subject: [PATCH 4/9] Added test global token --- .yamato/upm-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index f5e25057f8..d12e03fa8b 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -27,7 +27,7 @@ # Run upm-ci verification tests as well as tests contained in the package. - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION% {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" - --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.stg.unity3d.com;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} + --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.stg.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} after: - {{ instabilities_install_win }} - {{ instabilities_run_win }} From 41b31b01e823fb5584d71005dbbf8abb53b5a17c Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 16 May 2025 10:21:36 +0100 Subject: [PATCH 5/9] Revert to using the updated win10 image --- .yamato/config.metadata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/config.metadata b/.yamato/config.metadata index 602c324d41..565df3b2ac 100644 --- a/.yamato/config.metadata +++ b/.yamato/config.metadata @@ -19,7 +19,7 @@ upm_ci_install: npm install -g upm-ci-utils@stable --registry https://artifactor platforms_win: - name: win type: Unity::VM - image: pets-ops/draft:M97c169bdfd16-4006556-1 + image: package-ci/win10:v4 flavor: b1.large - name: win_standalone type: Unity::VM From 44cb2cd2f835de1fc15ea7b1e220563aa89ec911 Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 16 May 2025 10:35:10 +0100 Subject: [PATCH 6/9] Re-enabled project coverage --- .yamato/upm-ci.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index d12e03fa8b..87c7a88ee0 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -26,8 +26,24 @@ - upm-ci package pack --package-path ./Packages/com.unity.inputsystem/ # Run upm-ci verification tests as well as tests contained in the package. - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION% - {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" - --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.stg.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}};flags:{{platform.name}}_{{editor.version}}\"" {% endif %} + {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.stg.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}}_pkg;flags:{{platform.name}}_{{editor.version}}_pkg\"" {% endif %} + {% if platform.installscript %} + - {{ unity_downloader_install }} + - {{ platform.installscript }} %EDITOR_VERSION% + {% endif %} + # ADBv2 on 2019.4 causes the test runner to not start on initial import when the + # samples are in the package. Move the samples back into the project. + - move /Y .\Packages\com.unity.inputsystem\Samples .\Assets + - move /Y .\Packages\com.unity.inputsystem\Samples.meta .\Assets + # Now run our full test suite that sits in Assets/Tests by running UTR on our project. + - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if platform.name == "win" and category.name == "functional" %}--enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="hostUrl:https://codecov.stg.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %} + # Merge all the code coverage results into one report. + {% if platform.name == "win" and category.name == "functional" %} + - '{{ yamato_source_dir_win }}/.Editor/Unity.exe -projectPath {{ yamato_source_dir_win }} -batchmode -debugCodeOptimization -enableCodeCoverage -coverageResultsPath {{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/ -coverageOptions "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;sourcePaths:{{ yamato_source_dir_win }}/Packages/com.unity.inputsystem/" -quit' + # Delete the Package and Project reports, only keep the final merged one to keep Aritfacts.zip smaller + - rmdir /s/q "{{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package" + - rmdir /s/q "{{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project" + {% endif %} after: - {{ instabilities_install_win }} - {{ instabilities_run_win }} From 27531afc518161c563d4bc11914e08acf0505072 Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 16 May 2025 11:18:47 +0100 Subject: [PATCH 7/9] Finally testing on production Codecov --- .yamato/upm-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 87c7a88ee0..b705d9b170 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -26,7 +26,7 @@ - upm-ci package pack --package-path ./Packages/com.unity.inputsystem/ # Run upm-ci verification tests as well as tests contained in the package. - upm-ci package test --package-path ./Packages/com.unity.inputsystem/ -u %EDITOR_VERSION% - {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"hostUrl:https://codecov.stg.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}}_pkg;flags:{{platform.name}}_{{editor.version}}_pkg\"" {% endif %} + {% if platform.name == "win" and category.name == "functional" %} --enable-code-coverage --code-coverage-options "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;pathReplacePatterns:@*,,Library/PackageCache,Packages" --extra-utr-arg="--coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Package --coverage-upload-options=\"reportsDir:upm-ci~/test-results/CodeCoverage/Package;name:{{platform.name}}_{{editor.version}}_pkg;flags:{{platform.name}}_{{editor.version}}_pkg\"" {% endif %} {% if platform.installscript %} - {{ unity_downloader_install }} - {{ platform.installscript }} %EDITOR_VERSION% @@ -36,7 +36,7 @@ - move /Y .\Packages\com.unity.inputsystem\Samples .\Assets - move /Y .\Packages\com.unity.inputsystem\Samples.meta .\Assets # Now run our full test suite that sits in Assets/Tests by running UTR on our project. - - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if platform.name == "win" and category.name == "functional" %}--enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="hostUrl:https://codecov.stg.unity3d.com;token:%CODECOV_TOKEN_TEST%;reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %} + - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% else %} --category=!Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} {% if category.name == "performance" %} --report-performance-data --performance-project-id=InputSystem {% endif %} {% if platform.name == "win" and category.name == "functional" %}--enable-code-coverage --coverage-options="generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem" --coverage-results-path={{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/Project --coverage-upload-options="reportsDir:upm-ci~/test-results/CodeCoverage/Project;name:{{platform.name}}_{{editor.version}}_project;flags:{{platform.name}}_{{editor.version}}_project" {% endif %} # Merge all the code coverage results into one report. {% if platform.name == "win" and category.name == "functional" %} - '{{ yamato_source_dir_win }}/.Editor/Unity.exe -projectPath {{ yamato_source_dir_win }} -batchmode -debugCodeOptimization -enableCodeCoverage -coverageResultsPath {{ yamato_source_dir_win }}/upm-ci~/test-results/CodeCoverage/ -coverageOptions "generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.InputSystem;sourcePaths:{{ yamato_source_dir_win }}/Packages/com.unity.inputsystem/" -quit' From fd23bccdd6fe6eb000a9a82ab3cc2f6165245129 Mon Sep 17 00:00:00 2001 From: Nikos Date: Fri, 16 May 2025 16:01:44 +0100 Subject: [PATCH 8/9] Enabled comments in codecov.yml --- .github/codecov.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index a3d69e87f8..bd07a149a6 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -2,7 +2,7 @@ codecov: max_report_age: false require_ci_to_pass: false notify: - wait_for_ci: false + wait_for_ci: true coverage: precision: 2 @@ -13,8 +13,22 @@ coverage: patch: false default_rules: flag_coverage_not_uploaded_behavior: exclude + project: + default: + target: auto + threshold: 1% + base: auto + if_ci_failed: success + informational: false + only_pulls: true -comment: false +comment: + layout: "diff,flags,files" + behavior: default + require_changes: false + require_base: true + require_head: true + after_n_builds: 1 flag_management: default_rules: From ffd02a47aed12591635dbb7f70908f5828fe098d Mon Sep 17 00:00:00 2001 From: Nikos Date: Fri, 16 May 2025 16:02:37 +0100 Subject: [PATCH 9/9] Enabled patch coverage in codecov.yml --- .github/codecov.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/codecov.yml b/.github/codecov.yml index bd07a149a6..bf5d7de884 100644 --- a/.github/codecov.yml +++ b/.github/codecov.yml @@ -9,8 +9,7 @@ coverage: round: down range: "50...70" status: - project: false - patch: false + patch: true default_rules: flag_coverage_not_uploaded_behavior: exclude project: