chore: Create and use CMB Service Yamato job (#3445)

<!-- Replace this block with what this PR does and why. Describe what
you'd like reviewers to know, how you applied the engineering
principles, and any interesting tradeoffs made. Delete bullet points
below that don't apply, and update the changelog section as appropriate.
-->

<!-- Add short version of the JIRA ticket to the PR title (e.g. "feat:
new shiny feature [MTT-123]") -->

## Changelog

- Added: A separate yamato job for running the CMB Service tests

## Testing and Documentation

- Updated pr trigger to run all standalone windows tests and then cmb
service ubuntu tests.

<!-- Uncomment and mark items off with a * if this PR deprecates any
API:
### Deprecated API
- [ ] An `[Obsolete]` attribute was added along with a `(RemovedAfter
yyyy-mm-dd)` entry.
- [ ] An [api updater] was added.
- [ ] Deprecation of the API is explained in the CHANGELOG.
- [ ] The users can understand why this API was removed and what they
should use instead.
-->

## Backport

<!-- If this is a backport:
 - Add the following to the PR title: "\[Backport\] ..." .
 - Link to the original PR.
If this needs a backport - state this here
If a backport is not needed please provide the reason why.
If the "Backports" section is not present it will lead to a CI test
failure.
-->

No backport required as this is related to cmb service which is 2.X only

Author: EmandM

.yamato/_triggers.yml

@@ -50,13 +50,14 @@ pull_request_trigger:
     - .yamato/_run-all.yml#run_all_package_tests_trunk
     # Run package EditMode and Playmode package tests on minimum supported editor (6000.0 in case of NGOv2.X)
     - .yamato/_run-all.yml#run_all_package_tests_6000
-    # Run project EditMode and PLaymode project tests on trunk 
+    # Run project EditMode and Playmode project tests on trunk
     - .yamato/_run-all.yml#run_all_project_tests_trunk
-    # Run project EditMode and PLaymode project tests on minimum supported editor (6000.0 in case of NGOv2.X)
+    # Run project EditMode and Playmode project tests on minimum supported editor (6000.0 in case of NGOv2.X)
     - .yamato/_run-all.yml#run_all_project_tests_6000
-    # Run standalone test. We run it only on Ubuntu since it's the fastest machine, and it was noted that for example distribution on macOS is taking 40m since we switched to Apple Silicon
+    # Run standalone test. Run windows standalone tests as this is our most common platform, and run ubuntu rust tests.
     # Coverage on other standalone machines is present in Nightly job so it's enough to not run all of them for PRs
-    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_ubuntu_il2cpp_6000.0
+    - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_win_il2cpp_6000.0
+    - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_6000.0
   triggers:
     cancel_old_ci: true
     pull_requests:
@@ -133,4 +134,4 @@ develop_weekly_trunk:
     # Build player for webgl platform on trunk
     - .yamato/_run-all.yml#run_all_webgl_builds
     # Run code coverage test
-    - .yamato/code-coverage.yml#code_coverage_ubuntu_trunk
+    - .yamato/code-coverage.yml#code_coverage_ubuntu_trunk

.yamato/cmb-service-standalone-tests.yml

@@ -0,0 +1,70 @@
+{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
+---
+
+# DESCRIPTION--------------------------------------------------------------------------
+  # This job is responsible test validation against the CMB Service.
+  # This job runs on the default test_platform which is Ubuntu.
+  # This job runs the standalone desktop tests and ignores all tests that are not related to the CMB Service.
+  # Those tests cover both PlayMode and EditMode tests from package test assemblies.
+  # This is only needed in NGOv2.X since NGOv1.X does not use the CMB Service.
+
+# CONFIGURATION STRUCTURE--------------------------------------------------------------
+  # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency:
+    # 1. For all desktop platform (Windows, macOS, Ubuntu)
+    # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors)
+    # 3. For the default project.
+    # 4. For all scripting backends (mono, il2cpp)
+
+# TECHNICAL CONSIDERATIONS---------------------------------------------------------------
+  # For desktop devices a split is into two phases is not required but we use it for consistency with setup of others standalone platforms:
+    # 1. Build Phase: Creates standalone players for desktop platforms
+    # 2. Run Phase: Executes runtime tests on actual desktop devices
+  # The Run phase uses build job as dependency
+
+  # Note: The build phase for this test is imported from the desktop standalone tests as the CMB Service test is a subset of the desktop standalone tests.
+
+#-----------------------------------------------------------------------------------
+
+
+# CMB SERVICE RUN PHASE CONFIGURATION------------------------------------------------------------------------------------
+{% for project in projects.default -%}
+{% for platform in test_platforms.default -%}
+{% for editor in validation_editors.all -%}
+{% for backend in scripting_backends -%}
+cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}:
+  name : CMB Service Test - NGO {{ project.name }} - [{{ platform.name }}, {{ editor }}, {{ backend }}]
+  agent:
+    type: {{ platform.type }}::GPU
+    image: {{ platform.image }}
+    flavor: {{ platform.flavor }}
+
+  variables:
+    # The echo server is a "mock" server that is only used to test encoding/decoding of messages.
+    # It is used by the DistributedAuthorityCodecTests. These are tests that are built and maintained by the CMB service team.
+    ECHO_SERVER_PORT: "7788"
+    # Set this to ensure the DistributedAuthorityCodecTests will fail if they cannot connect to the echo server.
+    # The default is to ignore the codec tests if the echo server fails to connect
+    ENSURE_CODEC_TESTS: "true"
+
+    # When USE_CMB_SERVICE is set to true, any C# tests configured to use the DA host will instead use the CMB service.
+    USE_CMB_SERVICE: "true"
+    # This is the port on which to run the full standalone CMB service.
+    # The port needs to be different from the echo server port as two processes cannot bind to same port.
+    CMB_SERVICE_PORT: "7799"
+
+  commands:
+    # run_cmb_service.sh builds and starts a release version of the full CMB service (along with the limited echo server)
+    - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT
+
+    - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
+    - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
+  artifacts:
+    logs:
+      paths:
+        - "test-results/**/*"
+  dependencies:
+    - .yamato/desktop-standalone-tests.yml#desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}
+{% endfor -%}

.yamato/desktop-standalone-tests.yml

@@ -77,30 +77,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
     model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile). In this case it's specifically for macOS (if used for win/ubuntu then it will cause rust server to fail connecting)
 {% endif %}
 
-# Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority)
-{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606
-  variables:
-    # The echo server is a "mock" server that is only used to test encoding/decoding of messages.
-    # It is used by the DistributedAuthorityCodecTests. These are tests that are built and maintained by the CMB service team.
-    ECHO_SERVER_PORT: "7788"
-    # Set this to ensure the DistributedAuthorityCodecTests will fail if they cannot connect to the echo server.
-    # The default is to ignore the codec tests if the echo server fails to connect
-    ENSURE_CODEC_TESTS: "true"
-
-    # When USE_CMB_SERVICE is set to true, any C# tests configured to use the DA host will instead use the CMB service.
-    USE_CMB_SERVICE: "true"
-    # This is the port on which to run the full standalone CMB service.
-    # The port needs to be different from the echo server port as two processes cannot bind to same port.
-    CMB_SERVICE_PORT: "7799"
-{% endif %}
-
   commands:
-# If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority)
-{% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606
-    # run_cmb_service.sh builds and starts a release version of the full CMB service (along with the limited echo server)
-    - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT
-{% endif %}
-
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models
     - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
   artifacts: