chore: Small yamato tweaks to try speed up our PR trigger runs

.yamato/README.md

@@ -1,15 +1,14 @@
 # Netcode for GameObjects CI Documentation
 
 ## Overview
-This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. 
+This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects.
 Specifics of each test are described within related files (for example .yamato/package-tests.yml) and this file present high level overview related to overall implementation.
 
 ## Test Configurations
 CI related files are present inside .yamato/ folder and we can distinguish specific tests
 
 ### Helper jobs
 - `.yamato/package-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing.
-- `.yamato/project-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. This packs all packages of a given project.
 - `.yamato/_run-all.yml` responsible for grouping tests into groups for easier management (for example "all console tests").
 - `.yamato/_triggers.yml` responsible for defining triggers (PR, nightly, weekly etc.) and defining which tests to run.
 - `disable-burst-if-requested.py` responsible for helping to disable burst if needed.
@@ -77,4 +76,4 @@ Currently, the CI implementation supports the following platforms:
 
 ## Design Considerations
 In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations.
-The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.
+The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.

.yamato/_run-all.yml

@@ -15,10 +15,10 @@
 run_quick_checks:
   name: Run Quick Initial Checks
   dependencies:
-    - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu
+    # ensure the code is running to our current standards
     - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk
-    # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version
-    - .yamato/vetting-test.yml#vetting_test
+    # ensure the package can be packed at at all on at least one platform
+    - .yamato/package-pack.yml#package_pack_-_ngo_win
 
 
 # Runs all package tests
@@ -253,3 +253,44 @@ run_all_project_tests_console_standalone_6000:
     - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_6000.0
 {% endfor -%}
 {% endfor -%}
+
+
+# Runs all CMB service tests
+run_all_project_tests_cmb_service:
+  name: Run All CMB Service Tests
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.default -%}
+{% for editor in validation_editors.all -%}
+{% for backend in scripting_backends -%}
+    - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+
+# Runs all CMB service tests on trunk editor
+run_all_project_tests_cmb_service_trunk:
+  name: Run All CMB Service Tests [Trunk only]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.default -%}
+{% for editor in validation_editors.default -%}
+{% for backend in scripting_backends -%}
+    - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+
+# Runs all CMB service tests on mimimum supported editor (6000.0 in case of NGOv2.X)
+run_all_project_tests_cmb_service_6000:
+  name: Run All CMB Service Tests [6000.0]
+  dependencies:
+{% for project in projects.default -%}
+{% for platform in test_platforms.default -%}
+{% for backend in scripting_backends -%}
+    - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_6000.0
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}

.yamato/_triggers.yml

@@ -44,15 +44,15 @@
   # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job.
 
 
-#-----------------------------------------------------------------------------------  
+#-----------------------------------------------------------------------------------
 
 # After some experimenting with CI setups we discovered that even though sometimes we don't need CI to run (no reason to run package tests if only Documentation is changed) there are some checks that devs may not realize but changes in seemingly unrelated files will cause their failures
 # This trigger was created to ensure that ALL PRs run this minimal check even when we don't need to run full tests
 pr_minimal_required_checks:
   name: Minimal PR checks
   dependencies:
-    - .yamato/package-pack.yml#package_pack_-_ngo_win
     - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk
+    - .yamato/package-pack.yml#package_pack_-_ngo_win
   triggers:
     expression: |-
       (pull_request.comment eq "ngo" OR
@@ -72,6 +72,9 @@ pr_code_changes_checks:
   name: Code changes PR checks
   # Run the following tests on a selection of different desktop platforms
   dependencies:
+    # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version
+    - .yamato/vetting-test.yml#vetting_test
+
     # Run package EditMode and Playmode package tests on trunk and an older supported editor (6000.0)
     - .yamato/package-tests.yml#package_test_-_ngo_trunk_mac
     - .yamato/package-tests.yml#package_test_-_ngo_6000.0_win
@@ -82,7 +85,7 @@ pr_code_changes_checks:
 
     # Run standalone test. We run it only on Ubuntu since it's the fastest machine, and it was noted that for example distribution on macOS is taking 40m since we switched to Apple Silicon
     # Coverage on other standalone machines is present in Nightly job so it's enough to not run all of them for PRs
-    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_win_il2cpp_6000.0
+    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_ubuntu_il2cpp_trunk
     - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_trunk
   triggers:
     expression: |-
@@ -104,9 +107,9 @@ pr_code_changes_checks:
         "**/*.md"
       ]
     cancel_old_ci: true
-      
-      
-      
+
+
+
 
 
 
@@ -144,6 +147,9 @@ develop_nightly:
     # Build player for webgl platform on trunk and 6000.0 editors
     - .yamato/_run-all.yml#run_all_webgl_builds_trunk
     - .yamato/_run-all.yml#run_all_webgl_builds_6000
+    # Run Runtime tests against cmb service on trunk and 6000.0 editors
+    - .yamato/_run-all.yml#run_all_project_tests_cmb_service_trunk
+    - .yamato/_run-all.yml#run_all_project_tests_cmb_service_6000
     # Build player for webgl platform on trunk and 6000.0 editors
     - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_ubuntu_trunk
     - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_6000.0
@@ -177,5 +183,7 @@ develop_weekly_trunk:
     - .yamato/_run-all.yml#run_all_project_tests_console_standalone
     # Build player for webgl platform on trunk
     - .yamato/_run-all.yml#run_all_webgl_builds
+    # Run Runtime tests against CMB service
+    - .yamato/_run-all.yml#run_all_project_tests_cmb_service
     # Run code coverage test
     - .yamato/code-coverage.yml#code_coverage_ubuntu_trunk

.yamato/cmb-service-standalone-tests.yml

@@ -57,7 +57,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}
     - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT
 
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
-    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
+    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800
   artifacts:
     logs:
       paths:

.yamato/code-coverage.yml

@@ -1,14 +1,14 @@
 {% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
 ---
-  
+
 # DESCRIPTION--------------------------------------------------------------------------
   # This job is responsible for executing package tests with code coverage analysis enabled.
   # Coverage analysis provides insights into:
     # Test coverage metrics for NGO assemblies
     # Line and branch coverage statistics
     # Generated HTML reports for coverage visualization
     # Additional metrics for coverage analysis
-  
+
 # CONFIGURATION STRUCTURE--------------------------------------------------------------
   # Jobs are generated using nested loops through:
     # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms)
@@ -19,12 +19,12 @@
   # Requires Unity Editor installation
   # Burst compilation is disabled to ensure accurate coverage measurement
   # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible
-  
+
 # QUALITY CONSIDERATIONS--------------------------------------------------------------------
   # To see where this job is included (in trigger job definitions) look into _triggers.yml file
-    
-    
-    
+
+
+
 {% for platform in test_platforms.default -%}
 {% for editor in validation_editors.default -%}
 code_coverage_{{ platform.name }}_{{ editor }}:
@@ -39,12 +39,12 @@ code_coverage_{{ platform.name }}_{{ editor }}:
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
     - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor
-    - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results
+    - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --rerun-strategy=Test --retry=1 --clean-library-on-rerun --artifacts-path=test-results
   artifacts:
     logs:
       paths:
         - "test-results/**/*"
   dependencies:
     - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}
 {% endfor -%}
-{% endfor -%}
+{% endfor -%}

.yamato/console-standalone-test.yml

@@ -4,7 +4,7 @@
 # DESCRIPTION--------------------------------------------------------------------------
   # This job is responsible for Console platform test validation.
   # Those tests cover both PlayMode and EditMode tests from package test assemblies.
-  
+
 # CONFIGURATION STRUCTURE--------------------------------------------------------------
   # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency:
     # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne)
@@ -16,10 +16,10 @@
     # 1. Build Phase: Creates standalone players for console platforms
     # 2. Run Phase: Executes runtime tests on actual console devices
   # The Run phase uses build job as dependency
-  
+
   # Note: More of a Unity specific but test assemblies need to be included in the build phase command
   # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices
-  
+
 # PLATFORM SPECIFICS-----------------------------------------------------------------
   # Common Requirements:
     # All consoles require IL2CPP scripting backend
@@ -29,13 +29,13 @@
     # Switch: ARM64 architecture only
     # Other Consoles: x64 architecture
     # Each console requires specific SDK paths and tools
-  
+
 # QUALITY THOUGHTS--------------------------------------------------------------------
   # TODO: consider adding all projects that have tests
   # To see where this job is included (in trigger job definitions) look into _triggers.yml file
 
-  
-  
+
+
 # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_build -%}
@@ -51,7 +51,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
 {% endif %}
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
-    - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800
+    - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --clean-library-on-rerun --build-only --timeout=1800
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -72,9 +72,9 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
-  
-  
-  
+
+
+
 # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.console_test -%}
@@ -90,7 +90,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
 {% endif %}
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
-    - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
+    - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800
   variables:
 # PS4 related
     SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -109,4 +109,4 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}:
     - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
 {% endfor -%}
 {% endfor -%}
-{% endfor -%}
+{% endfor -%}

.yamato/desktop-standalone-tests.yml

@@ -44,7 +44,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% endif %}
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %}
-    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800
+    - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --clean-library-on-rerun --build-only --timeout=1800
   artifacts:
     players:
       paths:
@@ -53,7 +53,8 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
       paths:
         - "artifacts/**/*"
   dependencies:
-    - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }}
+    - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors
+    - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
@@ -79,7 +80,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 
   commands:
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
-    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
+    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800
   artifacts:
     logs:
       paths: