From 146e9cc43724bcbda712c471fd910da269f3eee1 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 13:54:52 -0400 Subject: [PATCH 01/27] chore: Update NetworkSceneManagerEventNotifications to work with CMB service --- .../NetworkSceneManagerEventNotifications.cs | 151 +++++++++++------- 1 file changed, 94 insertions(+), 57 deletions(-) diff --git a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs index b641a1a3e6..3bf95d0ac5 100644 --- a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs +++ b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs @@ -1,6 +1,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Text; using NUnit.Framework; using Unity.Netcode; using Unity.Netcode.TestHelpers.Runtime; @@ -26,12 +27,7 @@ public class NetworkSceneManagerEventNotifications : NetcodeIntegrationTest private LoadSceneMode m_LoadSceneMode; private bool m_CanStartServerOrClients = false; private bool m_LoadEventCompleted = false; - - // TODO: [CmbServiceTests] Adapt to run with the service - protected override bool UseCMBService() - { - return false; - } + private NetworkManager m_ClientToTestLoading; internal class SceneTestInfo { @@ -78,11 +74,16 @@ protected override IEnumerator OnTearDown() protected override IEnumerator OnStartedServerAndClients() { - m_ServerNetworkManager.SceneManager.OnSceneEvent += ServerSceneManager_OnSceneEvent; - foreach (var client in m_ClientNetworkManagers) + foreach (var manager in m_NetworkManagers) { - client.SceneManager.ClientSynchronizationMode = m_LoadSceneMode; - client.SceneManager.OnSceneEvent += ClientSceneManager_OnSceneEvent; + if (manager.IsServer || manager.LocalClient.IsSessionOwner) + { + manager.SceneManager.OnSceneEvent += ServerSceneManager_OnSceneEvent; + continue; + } + + manager.SceneManager.ClientSynchronizationMode = m_LoadSceneMode; + manager.SceneManager.OnSceneEvent += ClientSceneManager_OnSceneEvent; } return base.OnStartedServerAndClients(); } @@ -93,17 +94,19 @@ private void ClientSceneManager_OnSceneEvent(SceneEvent sceneEvent) { // Validate that the clients finish synchronization and they used the proper synchronization mode case SceneEventType.SynchronizeComplete: - { - var matchedClient = m_ClientNetworkManagers.Where(c => c.LocalClientId == sceneEvent.ClientId); - Assert.True(matchedClient.Count() > 0, $"Found no client {nameof(NetworkManager)}s that had a {nameof(NetworkManager.LocalClientId)} of {sceneEvent.ClientId}"); - Assert.AreEqual(matchedClient.First().SceneManager.ClientSynchronizationMode, m_ServerNetworkManager.SceneManager.ClientSynchronizationMode); - break; - } + { + var authority = GetAuthorityNetworkManager(); + var matchedClient = m_ClientNetworkManagers.FirstOrDefault(c => c.LocalClientId == sceneEvent.ClientId); + Assert.That(matchedClient, Is.Not.Null, $"Found no client {nameof(NetworkManager)}s that had a {nameof(NetworkManager.LocalClientId)} of {sceneEvent.ClientId}"); + Assert.AreEqual(matchedClient.SceneManager.ClientSynchronizationMode, authority.SceneManager.ClientSynchronizationMode); + break; + } } } private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) { + var authority = GetAuthorityNetworkManager(); VerboseDebug($"[SceneEvent] ClientId:{sceneEvent.ClientId} | EventType: {sceneEvent.SceneEventType}"); switch (sceneEvent.SceneEventType) { @@ -126,16 +129,16 @@ private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) } case SceneEventType.LoadComplete: { - if (sceneEvent.ClientId == NetworkManager.ServerClientId) + if (sceneEvent.ClientId == authority.LocalClientId) { var scene = sceneEvent.Scene; m_CurrentScene = scene; } - if (sceneEvent.ClientId == m_ClientNetworkManagers[0].LocalClientId) + if (sceneEvent.ClientId == m_ClientToTestLoading.LocalClientId) { if (!m_ScenesLoaded.Contains(sceneEvent.SceneName)) { - Debug.Log($"Loaded {sceneEvent.SceneName}"); + VerboseLog($"Loaded {sceneEvent.SceneName}"); m_ScenesLoaded.Add(sceneEvent.SceneName); } } @@ -162,10 +165,10 @@ private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) // If we are a server and this is being processed by the server, then add the server to the completed list // to validate that the event completed on all clients (and the server). - if (!m_ServerNetworkManager.IsHost && sceneEvent.ClientId == m_ServerNetworkManager.LocalClientId && - !sceneEvent.ClientsThatCompleted.Contains(m_ServerNetworkManager.LocalClientId)) + if (!authority.IsHost && sceneEvent.ClientId == authority.LocalClientId && + !sceneEvent.ClientsThatCompleted.Contains(authority.LocalClientId)) { - sceneEvent.ClientsThatCompleted.Add(m_ServerNetworkManager.LocalClientId); + sceneEvent.ClientsThatCompleted.Add(authority.LocalClientId); } if (sceneEvent.SceneEventType == SceneEventType.LoadEventCompleted) { @@ -180,16 +183,16 @@ private void ServerSceneManager_OnSceneEvent(SceneEvent sceneEvent) } private void SceneManager_OnSceneEvent(SceneEvent sceneEvent) { - Debug.Log($"[SceneEvent] ClientId:{sceneEvent.ClientId} | EventType: {sceneEvent.SceneEventType}"); + VerboseLog($"[SceneEvent] ClientId:{sceneEvent.ClientId} | EventType: {sceneEvent.SceneEventType}"); switch (sceneEvent.SceneEventType) { case SceneEventType.LoadComplete: { - if (sceneEvent.ClientId == m_ClientNetworkManagers[0].LocalClientId) + if (sceneEvent.ClientId == m_ClientToTestLoading.LocalClientId) { if (!m_ScenesLoaded.Contains(sceneEvent.SceneName)) { - Debug.Log($"Loaded {sceneEvent.SceneName}"); + VerboseLog($"Loaded {sceneEvent.SceneName}"); m_ScenesLoaded.Add(sceneEvent.SceneName); } } @@ -198,11 +201,11 @@ private void SceneManager_OnSceneEvent(SceneEvent sceneEvent) case SceneEventType.UnloadComplete: { - if (sceneEvent.ClientId == m_ClientNetworkManagers[0].LocalClientId) + if (sceneEvent.ClientId == m_ClientToTestLoading.LocalClientId) { if (m_ScenesLoaded.Contains(sceneEvent.SceneName)) { - Debug.Log($"Unloaded {sceneEvent.SceneName}"); + VerboseLog($"Unloaded {sceneEvent.SceneName}"); // We check here for single mode because the final scene event // will be SceneEventType.LoadEventCompleted (easier to trap for it here) m_ScenesLoaded.Remove(sceneEvent.SceneName); @@ -226,26 +229,29 @@ protected override bool CanStartServerAndClients() [UnityTest] public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadSceneMode) { + var authority = GetAuthorityNetworkManager(); + var nonAuthority = GetNonAuthorityNetworkManager(); + m_ClientToTestLoading = nonAuthority; m_LoadSceneMode = loadSceneMode; m_CurrentSceneName = k_SceneToLoad; m_CanStartServerOrClients = true; yield return StartServerAndClients(); - yield return WaitForConditionOrTimeOut(() => m_ClientsReceivedSynchronize.Count == (m_ClientNetworkManagers.Length)); - Assert.False(s_GlobalTimeoutHelper.TimedOut, $"Timed out waiting for all clients to receive synchronization event! Received: {m_ClientsReceivedSynchronize.Count} | Expected: {m_ClientNetworkManagers.Length}"); + yield return WaitForConditionOrTimeOut(() => m_ClientsReceivedSynchronize.Count == NumberOfClients); + AssertOnTimeout($"Timed out waiting for all clients to receive synchronization event! Received: {m_ClientsReceivedSynchronize.Count} | Expected: {NumberOfClients}"); if (loadSceneMode == LoadSceneMode.Single) { - m_ClientNetworkManagers[0].SceneManager.OnSceneEvent += SceneManager_OnSceneEvent; + m_ClientToTestLoading.SceneManager.OnSceneEvent += SceneManager_OnSceneEvent; } // Now prepare for the scene testing InitializeSceneTestInfo(); // Test loading scenes and the associated event messaging and notification pipelines ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); // Check error status for trying to load during an already in progress scene event - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.SceneEventInProgress); + Assert.AreEqual(authority.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.SceneEventInProgress); // Wait for all clients to load the scene yield return WaitForConditionOrTimeOut(ConditionPassed); @@ -262,7 +268,7 @@ public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadScene m_CurrentSceneName = k_InSceneNetworkObject; ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(k_InSceneNetworkObject, LoadSceneMode.Additive), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.LoadScene(k_InSceneNetworkObject, LoadSceneMode.Additive), SceneEventProgressStatus.Started); // Wait for all clients to additively load this additional scene yield return WaitForConditionOrTimeOut(ConditionPassed); @@ -272,40 +278,39 @@ public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadScene // Now single mode load a new scene (i.e. "scene switch") m_CurrentSceneName = k_BaseUnitTestSceneName; ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.LoadScene(m_CurrentSceneName, loadSceneMode), SceneEventProgressStatus.Started); // Wait for all clients to perform scene switch yield return WaitForConditionOrTimeOut(ConditionPassed); AssertOnTimeout($"Timed out waiting for all clients to switch to scene {m_CurrentSceneName}!"); // Make sure the server scene is the active scene SceneManager.SetActiveScene(m_CurrentScene); - yield return WaitForConditionOrTimeOut(() => !m_ScenesLoaded.Contains(k_SceneToLoad) && !m_ScenesLoaded.Contains(k_InSceneNetworkObject)); - var additionalInfo = string.Empty; - if (s_GlobalTimeoutHelper.TimedOut) + var helper = new TimeoutHelper(); + yield return WaitForConditionOrTimeOut(() => !m_ScenesLoaded.Contains(k_SceneToLoad) && !m_ScenesLoaded.Contains(k_InSceneNetworkObject), helper); + var additionalInfo = new StringBuilder(); + if (helper.TimedOut) { - foreach (var sceneName in m_ScenesLoaded) - { - additionalInfo += $"{sceneName},"; - } - Debug.Break(); + additionalInfo.Append("Scenes currently loaded: ["); + additionalInfo.AppendJoin(", ", m_ScenesLoaded); + additionalInfo.Append("]"); } - AssertOnTimeout($"{nameof(m_ScenesLoaded)} still contains some of the scenes that were expected to be unloaded!\n {additionalInfo}"); + AssertOnTimeout($"{nameof(m_ScenesLoaded)} still contains some of the scenes that were expected to be unloaded!\n {additionalInfo}", helper); } // Test unloading additive scenes and the associated event messaging and notification pipelines ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.Started); + Assert.AreEqual(authority.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.Started); yield return WaitForConditionOrTimeOut(ConditionPassed); AssertOnTimeout($"Timed out waiting for all clients to unload {m_CurrentSceneName}!"); // Check error status for trying to unloading something not loaded ResetWait(); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.SceneNotLoaded); + Assert.AreEqual(authority.SceneManager.UnloadScene(m_CurrentScene), SceneEventProgressStatus.SceneNotLoaded); // Check error status for trying to load an invalid scene name LogAssert.Expect(LogType.Error, $"Scene '{k_InvalidSceneName}' couldn't be loaded because it has not been added to the build settings scenes in build list."); - Assert.AreEqual(m_ServerNetworkManager.SceneManager.LoadScene(k_InvalidSceneName, LoadSceneMode.Additive), SceneEventProgressStatus.InvalidSceneName); + Assert.AreEqual(authority.SceneManager.LoadScene(k_InvalidSceneName, LoadSceneMode.Additive), SceneEventProgressStatus.InvalidSceneName); } @@ -329,9 +334,7 @@ private void ResetWait() /// private void InitializeSceneTestInfo() { - m_ShouldWaitList.Add(new SceneTestInfo() { ClientId = NetworkManager.ServerClientId, ShouldWait = false }); - - foreach (var manager in m_ClientNetworkManagers) + foreach (var manager in m_NetworkManagers) { m_ShouldWaitList.Add(new SceneTestInfo() { ClientId = manager.LocalClientId, ShouldWait = false }); } @@ -343,12 +346,20 @@ private void InitializeSceneTestInfo() /// private bool ConditionPassed() { - var completed = true; - if (m_LoadSceneMode == LoadSceneMode.Single) + if (m_LoadSceneMode == LoadSceneMode.Single && !m_LoadEventCompleted) { - completed = m_LoadEventCompleted; + return false; } - return completed && !(m_ShouldWaitList.Select(c => c).Where(c => c.ProcessedEvent != true && c.ShouldWait == true).Count() > 0); + + foreach (var client in m_ShouldWaitList) + { + if (!client.ProcessedEvent || client.ShouldWait) + { + return false; + } + } + + return true; } /// @@ -356,7 +367,15 @@ private bool ConditionPassed() /// private bool ContainsClient(ulong clientId) { - return m_ShouldWaitList.Select(c => c.ClientId).Where(c => c == clientId).Count() > 0; + foreach (var client in m_ShouldWaitList) + { + if (client.ClientId == clientId) + { + return true; + } + } + + return false; } /// @@ -364,7 +383,13 @@ private bool ContainsClient(ulong clientId) /// private void SetClientProcessedEvent(ulong clientId) { - m_ShouldWaitList.Select(c => c).Where(c => c.ClientId == clientId).First().ProcessedEvent = true; + foreach (var client in m_ShouldWaitList) + { + if (client.ClientId == clientId) + { + client.ProcessedEvent = true; + } + } } /// @@ -372,9 +397,13 @@ private void SetClientProcessedEvent(ulong clientId) /// private void SetClientWaitDone(List clients) { - foreach (var clientId in clients) + var lookup = clients.ToHashSet(); + foreach (var client in m_ShouldWaitList) { - m_ShouldWaitList.Select(c => c).Where(c => c.ClientId == clientId).First().ShouldWait = false; + if (lookup.Contains(client.ClientId)) + { + client.ShouldWait = false; + } } } @@ -401,5 +430,13 @@ private bool ContainsAllClients(List clients) } return true; } + + private void VerboseLog(string message) + { + if (OnSetVerboseDebug()) + { + Debug.Log(message); + } + } } } From 816fea9cee55184f6c6ab871569b4aff43e011da Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 18:29:05 -0400 Subject: [PATCH 02/27] chore: Small yamato tweaks to try speed up our PR trigger runs --- .yamato/_triggers.yml | 10 +++++----- .yamato/package-pack.yml | 2 +- .yamato/package-tests.yml | 14 +++++++------- .yamato/project-pack.yml | 19 ++++++++----------- .yamato/project-tests.yml | 20 ++++++++------------ .yamato/project.metafile | 4 ++++ 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index ec379d091b..b91c8bda8a 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -44,7 +44,7 @@ # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. -#----------------------------------------------------------------------------------- +#----------------------------------------------------------------------------------- # After some experimenting with CI setups we discovered that even though sometimes we don't need CI to run (no reason to run package tests if only Documentation is changed) there are some checks that devs may not realize but changes in seemingly unrelated files will cause their failures # This trigger was created to ensure that ALL PRs run this minimal check even when we don't need to run full tests @@ -82,7 +82,7 @@ pr_code_changes_checks: # Run standalone test. We run it only on Ubuntu since it's the fastest machine, and it was noted that for example distribution on macOS is taking 40m since we switched to Apple Silicon # Coverage on other standalone machines is present in Nightly job so it's enough to not run all of them for PRs - - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_win_il2cpp_6000.0 + - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_ubuntu_il2cpp_trunk - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_trunk triggers: expression: |- @@ -104,9 +104,9 @@ pr_code_changes_checks: "**/*.md" ] cancel_old_ci: true - - - + + + diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index c6c0643889..7d3c6cc716 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -29,7 +29,7 @@ package_pack_-_ngo_{{ platform.name }}: agent: type: {{ platform.type }} image: {{ platform.image }} - flavor: {{ platform.flavor }} + flavor: {{ platform.smaller_flavor }} {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index cbadbfba44..828e30ee5e 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -5,23 +5,23 @@ # This job is responsible for execution of package-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from package test assemblies # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) # Uses PVP for package validation. Specifically it looks for supported profiles which we should conform to but takes ./pvpExceptions.json file into consideration where we note our known issues related to PVP checks - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file # TODO: we should aim to replace target PVP profile from supported to gold - + #------------------------------------------------------------------------------------ {% for platform in test_platforms.desktop -%} @@ -31,7 +31,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: agent: type: {{ platform.type }} image: {{ platform.image }} - flavor: {{ platform.flavor }} + flavor: {{ platform.smaller_flavor }} {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} @@ -40,7 +40,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: UNITY_EXT_LOGGING: 1 commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. - + # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results - upm-pvp require {% if platform.name == "win" %}"%XRAY_PROFILE%"{% else %}"$XRAY_PROFILE"{% endif %} --results pvp-results @@ -57,4 +57,4 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 647d395bde..46fde069b2 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -4,22 +4,20 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for packing a specific project. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects (testproject, minimalproject, testproject-tools-integration). # 2. For all desktop platforms (Win, Ubuntu, Mac) - + # TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- # Job does not require Unity Editor in order to perform packing. # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file - # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects - - -#-------------------------------------------------------------------------------------- + +#-------------------------------------------------------------------------------------- {% for project in projects.all -%} {% for platform in test_platforms.desktop -%} @@ -28,13 +26,12 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: agent: type: {{ platform.type }} image: {{ platform.image }} - flavor: {{ platform.flavor }} + flavor: {{ platform.smaller_flavor }} {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - upm-ci project pack --project-path {{ project.path }} + - upm-pvp pack --project {{ project.path }} --output upm-ci~/packages dependencies: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors artifacts: @@ -42,4 +39,4 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: paths: - "upm-ci~/packages/**/*" {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index a39f957480..88de775165 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -4,24 +4,23 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job executes project-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from project test assemblies - # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) - + # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) ?? + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects WITH TESTS (filtered by has_tests flag) (testproject, testproject-tools-interation) [For more info look into project.metafile configuration] # 2. For all desktop platforms (Windows, Ubuntu, macOS) # 3. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # This job runs in Editor context only (no player builds is required) # No scripting backend variations needed (Editor context) # Architecture variations not applicable (Editor context) # Requires project packaging as prerequisite (dependency job) - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ {% for project in projects.all -%} @@ -38,16 +37,13 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --reruncount=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: - - "upm-ci~/test-results/**/*" - dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - "test-results/**/*" {% endfor -%} {% endfor -%} {% endif -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index d6c93e86ff..b1e5bf3279 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -9,6 +9,7 @@ # type --> Specifies the Bokken agent type (e.g., Unity::VM, Unity::VM::osx, Unity::mobile::shield) # image --> Defines the package-ci Bokken image to use for the environment (e.g., package-ci/ubuntu-22.04:v4). This is basically a device configuration # flavor --> Determines the VM size/resources (e.g., b1.small, b1.large, m1.mac) + # smaller_flavor --> An override for flavor that determines the VM size/resources for lighter weight jobs that can run on a smaller vm # standalone --> Specifies the build target platform (e.g., StandaloneLinux64, Android, IOS) # model --> Defines specific hardware model requirements (e.g., rtx2080, iPhone model 13). Notice that trunk currently (19.08.2025) has 13.0 as minimal iOS version which devices below this are not supporting # base --> Indicates the base operating system for build operations (e.g., win, mac) @@ -45,18 +46,21 @@ test_platforms: type: Unity::VM image: package-ci/ubuntu-22.04:v4 flavor: b1.large + smaller_flavor: b1.medium standalone: StandaloneLinux64 model: rtx2080 - name: win type: Unity::VM image: package-ci/win10:v4 flavor: b1.large + smaller_flavor: b1.medium standalone: StandaloneWindows64 model: rtx2080 - name: mac type: Unity::VM::osx image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) flavor: m1.mac + smaller_flavor: m1.mac # mac doesn't have a smaller vm size. We define it anyway as it simplifies the yaml templating to have it defined. standalone: StandaloneOSX model: M1 # The default model (an x64 Intel Mac VM) quite often caused a known issue of doing all the bitflips in packages resulting in failures # For mobile devices there is a split between the build and run phase so there is a need of splitting specification for both From 790bc623cda73986b41ec0ebd87df97054042cbe Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 19:12:05 -0400 Subject: [PATCH 03/27] re-add the dependency on in --- .yamato/project-tests.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 88de775165..7d7d89de2a 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -4,7 +4,7 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job executes project-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from project test assemblies - # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) ?? + # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: @@ -37,12 +37,15 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --reruncount=1 --clean-library-on-rerun --timeout=1800 + - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci artifacts: logs: paths: - - "test-results/**/*" + - "upm-ci~/test-results/**/*" + dependencies: + - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} {% endfor -%} {% endif -%} From 2aa215ead1029d2fece6062543b2207f6d9aa2d0 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 19:26:52 -0400 Subject: [PATCH 04/27] Ensure CMB Service tests are running on nightly and weekly jobs --- .yamato/_run-all.yml | 41 +++++++++++++++++++++++++++++++++++++++++ .yamato/_triggers.yml | 5 +++++ 2 files changed, 46 insertions(+) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 089a8b5247..6ba0e948d0 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -253,3 +253,44 @@ run_all_project_tests_console_standalone_6000: - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_6000.0 {% endfor -%} {% endfor -%} + + +# Runs all CMB service tests +run_all_project_tests_cmb_service: + name: Run All CMB Service Tests + dependencies: +{% for project in projects.default -%} +{% for platform in test_platforms.default -%} +{% for editor in validation_editors.all -%} +{% for backend in scripting_backends -%} + - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }} +{% endfor -%} +{% endfor -%} +{% endfor -%} +{% endfor -%} + +# Runs all CMB service tests on trunk editor +run_all_project_tests_cmb_service_trunk: + name: Run All CMB Service Tests [Trunk only] + dependencies: +{% for project in projects.default -%} +{% for platform in test_platforms.default -%} +{% for editor in validation_editors.default -%} +{% for backend in scripting_backends -%} + - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }} +{% endfor -%} +{% endfor -%} +{% endfor -%} +{% endfor -%} + +# Runs all CMB service tests on mimimum supported editor (6000.0 in case of NGOv2.X) +run_all_project_tests_cmb_service_6000: + name: Run All CMB Service Tests [6000.0] + dependencies: +{% for project in projects.default -%} +{% for platform in test_platforms.default -%} +{% for backend in scripting_backends -%} + - .yamato/cmb_service_standalone_tests.yml#cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_6000.0 +{% endfor -%} +{% endfor -%} +{% endfor -%} diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index b91c8bda8a..927a6c04bb 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -144,6 +144,9 @@ develop_nightly: # Build player for webgl platform on trunk and 6000.0 editors - .yamato/_run-all.yml#run_all_webgl_builds_trunk - .yamato/_run-all.yml#run_all_webgl_builds_6000 + # Run Runtime tests against cmb service on trunk and 6000.0 editors + - .yamato/_run-all.yml#run_all_project_tests_cmb_service_trunk + - .yamato/_run-all.yml#run_all_project_tests_cmb_service_6000 # Build player for webgl platform on trunk and 6000.0 editors - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_ubuntu_trunk - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_6000.0 @@ -177,5 +180,7 @@ develop_weekly_trunk: - .yamato/_run-all.yml#run_all_project_tests_console_standalone # Build player for webgl platform on trunk - .yamato/_run-all.yml#run_all_webgl_builds + # Run Runtime tests against CMB service + - .yamato/_run-all.yml#run_all_project_tests_cmb_service # Run code coverage test - .yamato/code-coverage.yml#code_coverage_ubuntu_trunk From 5a240a9e52f785b3e0c59a85f7758ddaf1d0e2bc Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 19:30:56 -0400 Subject: [PATCH 05/27] Re-add update to project-tests --- .yamato/project-tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 7d7d89de2a..955aea1074 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -4,7 +4,7 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job executes project-specific tests in Unity Editor context # Those tests cover both PlayMode and EditMode tests from project test assemblies - # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) + # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) ?? # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: @@ -37,13 +37,12 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --reruncount=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: - - "upm-ci~/test-results/**/*" + - "test-results/**/*" dependencies: - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} From 96c96111eef23f8c92c0ff6e317add22ec5b5c52 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 19:44:22 -0400 Subject: [PATCH 06/27] Swap --clean-library-on-rerun to --clean-library to see if that'll allow us to only rerun failed tests --- .yamato/cmb-service-standalone-tests.yml | 2 +- .yamato/code-coverage.yml | 16 +++++++-------- .yamato/console-standalone-test.yml | 24 +++++++++++----------- .yamato/desktop-standalone-tests.yml | 4 ++-- .yamato/mobile-standalone-test.yml | 26 ++++++++++++------------ .yamato/package-tests.yml | 2 +- .yamato/performance-tests.yml | 12 +++++------ .yamato/project-tests.yml | 2 +- 8 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.yamato/cmb-service-standalone-tests.yml b/.yamato/cmb-service-standalone-tests.yml index 3a9cc76b5d..bfca8d2fd9 100644 --- a/.yamato/cmb-service-standalone-tests.yml +++ b/.yamato/cmb-service-standalone-tests.yml @@ -57,7 +57,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }} - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 688fa2e03f..1715395118 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -1,6 +1,6 @@ {% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- - + # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for executing package tests with code coverage analysis enabled. # Coverage analysis provides insights into: @@ -8,7 +8,7 @@ # Line and branch coverage statistics # Generated HTML reports for coverage visualization # Additional metrics for coverage analysis - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) @@ -19,12 +19,12 @@ # Requires Unity Editor installation # Burst compilation is disabled to ensure accurate coverage measurement # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - - + + + {% for platform in test_platforms.default -%} {% for editor in validation_editors.default -%} code_coverage_{{ platform.name }}_{{ editor }}: @@ -39,7 +39,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library --artifacts-path=test-results artifacts: logs: paths: @@ -47,4 +47,4 @@ code_coverage_{{ platform.name }}_{{ editor }}: dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 999651ef2c..98eecd9a7b 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -4,7 +4,7 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Console platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) @@ -16,10 +16,10 @@ # 1. Build Phase: Creates standalone players for console platforms # 2. Run Phase: Executes runtime tests on actual console devices # The Run phase uses build job as dependency - + # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # PLATFORM SPECIFICS----------------------------------------------------------------- # Common Requirements: # All consoles require IL2CPP scripting backend @@ -29,13 +29,13 @@ # Switch: ARM64 architecture only # Other Consoles: x64 architecture # Each console requires specific SDK paths and tools - + # QUALITY THOUGHTS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_build -%} @@ -51,7 +51,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library --build-only --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -72,9 +72,9 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endfor -%} {% endfor -%} {% endfor -%} - - - + + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} @@ -90,7 +90,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -109,4 +109,4 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 1b58afc459..11f3e916a1 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -44,7 +44,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library --build-only --timeout=1800 artifacts: players: paths: @@ -79,7 +79,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index e7c270948e..0a8f625702 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -4,13 +4,13 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible for Mobile platform test validation. # Those tests cover both PlayMode and EditMode tests from package test assemblies. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated using nested loops through: # 1. For all mobile platform (Android, iOS) # 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors) # 3. For the default project. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # For mobile devices a split is required into two phases: # 1. Build Phase: Creates standalone players for mobile platforms @@ -18,7 +18,7 @@ # The Run phase uses build job as dependency # Note: More of a Unity specific but test assemblies need to be included in the build phase command # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices - + # PLATFORM SPECIFICS-------------------------------------------------------------------- # iOS Requirements: # Must use IL2CPP scripting backend @@ -29,12 +29,12 @@ # Uses IL2CPP scripting backend (recommended over Mono) # Supports both ARM64 and ARMv7 architectures # Can be build on any desktop platform - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: consider adding all projects that have tests # To see where this job is included (in trigger job definitions) look into _triggers.yml file - - + + # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_build -%} @@ -48,14 +48,14 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% if platform.model %} model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - # Automatic UI interruption handling is available for iPhones running iOS 15 and above (models SE-Gen3 and 13). + # Automatic UI interruption handling is available for iPhones running iOS 15 and above (models SE-Gen3 and 13). # It is enabled by default when using those devices. Otherwise, system alerts (e.g. “Local Network Access” permission alert, introduced in iOS 14) might cause disruptions during test execution. # If building of the test app is done on a separate (“Build”) job, please make sure that that job has environment variable UNITY_HANDLEUIINTERRUPTIONS set to 1. variables: UNITY_HANDLEUIINTERRUPTIONS: 1 commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library --build-only --timeout=1800 artifacts: players: paths: @@ -66,8 +66,8 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endfor -%} {% endfor -%} {% endfor -%} - - + + # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} @@ -84,7 +84,7 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: commands: # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - + {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub @@ -97,11 +97,11 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library --timeout=3600 {% else %} # Run tests for non-Android devices - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% {% endif %} artifacts: logs: diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 828e30ee5e..8080c2e156 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -47,7 +47,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 --clean-library-on-rerun + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 --clean-library artifacts: logs: paths: diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index 26bf8102f4..ff06ff2ba7 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -5,21 +5,21 @@ # This job is responsible for executing performance tests for NGO package. # Its configuration is set to not report any data but just to give results (at least in current state since we don't have any tests to run). # Currently because of lack of performance tests this job will always return "no tests have been selected" and because oif that it's not included in any trigger jobs. - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all desktop platforms (Windows, Ubuntu, macOS) # 2. For all supported Unity Editor versions (For NGOv2.X it means 6000+) # 3. For the default project (project is used only as a context for the build). TODO-comment: if performance tests would be included in projects then we should make an approperiate split. - + # TECHNICAL CONSIDERATIONS--------------------------------------------------------------- # Tests are run in Editor context only # No performance metrics are reported to monitoring systems - + # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future. We should discuss how to approach the topic of performance testing # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ {% for platform in test_platforms.desktop -%} @@ -36,7 +36,7 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin {% endif %} commands: - unity-downloader-cli -u {{ editor }} -c Editor --wait {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor - - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata + - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library --dontreportperformancedata # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: logs: @@ -44,4 +44,4 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin - "artifacts/**/*" {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 955aea1074..549b83d2cd 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -38,7 +38,7 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --reruncount=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --reruncount=1 --clean-library --timeout=1800 artifacts: logs: paths: From 59463bfce8eb5f85c898827cc77d80a1cf43588c Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 19:46:12 -0400 Subject: [PATCH 07/27] Force NetworkSceneManagerEventNotifications to fail to test rerun behaviour --- .../NetworkSceneManagerEventNotifications.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs index 3bf95d0ac5..dba9897ab5 100644 --- a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs +++ b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs @@ -94,13 +94,13 @@ private void ClientSceneManager_OnSceneEvent(SceneEvent sceneEvent) { // Validate that the clients finish synchronization and they used the proper synchronization mode case SceneEventType.SynchronizeComplete: - { - var authority = GetAuthorityNetworkManager(); - var matchedClient = m_ClientNetworkManagers.FirstOrDefault(c => c.LocalClientId == sceneEvent.ClientId); - Assert.That(matchedClient, Is.Not.Null, $"Found no client {nameof(NetworkManager)}s that had a {nameof(NetworkManager.LocalClientId)} of {sceneEvent.ClientId}"); - Assert.AreEqual(matchedClient.SceneManager.ClientSynchronizationMode, authority.SceneManager.ClientSynchronizationMode); - break; - } + { + var authority = GetAuthorityNetworkManager(); + var matchedClient = m_ClientNetworkManagers.FirstOrDefault(c => c.LocalClientId == sceneEvent.ClientId); + Assert.That(matchedClient, Is.Not.Null, $"Found no client {nameof(NetworkManager)}s that had a {nameof(NetworkManager.LocalClientId)} of {sceneEvent.ClientId}"); + Assert.AreEqual(matchedClient.SceneManager.ClientSynchronizationMode, authority.SceneManager.ClientSynchronizationMode); + break; + } } } @@ -311,6 +311,7 @@ public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadScene // Check error status for trying to load an invalid scene name LogAssert.Expect(LogType.Error, $"Scene '{k_InvalidSceneName}' couldn't be loaded because it has not been added to the build settings scenes in build list."); Assert.AreEqual(authority.SceneManager.LoadScene(k_InvalidSceneName, LoadSceneMode.Additive), SceneEventProgressStatus.InvalidSceneName); + Assert.Fail("Test reruns!"); } From ecb9bce350fba78debf882a9d9524022d3dcb029 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 19:59:38 -0400 Subject: [PATCH 08/27] Run the lighter dotnet standards check first, before running the heavier unity standards check --- .yamato/project-standards.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 6a70daf149..aa68355e2a 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -8,24 +8,24 @@ # Project structure validation # Coding convention adherence # Solution file synchronization - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all NGO projects (testproject, testproject-tools-interation, minimalproject) # 2. For default platform only (Ubuntu) since standards would not vary between platforms (no need for checks on more platforms) # 3. For default editor version (trunk) since standards would not vary between editors (no need for checks on more editors) - + # TECHNICAL CONSTRAINTS--------------------------------------------------------------- # Requires .NET SDK installed (should be preinstalled on the image so we just check for version) # Needs Unity Editor for solution synchronization # Uses custom standards validation tool (netcode.standards) # Generates no test artifacts (pass/fail only). Eventual failure will be visible in the logs - + # QUALITY THOUGHTS-------------------------------------------------------------------- # While testproject validation would be sufficient, since it validates both project and package (where package is our main concern) jobs for all projects are being generated to ensure that we conform to quality standards in all projects. # TODO: consider modifying the approach and adding checks for minimal supported editor. In case of NGOv1.X it has proven to yield different results (But since NGOv2.X support starts from 6000.0 editor it's not that time pressuring) # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ {% for project in projects.all -%} @@ -44,10 +44,10 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: # .NET environment setup. Ensures required .NET SDK and formatting tools are available - dotnet --version - dotnet format --version - + - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check + - unity-downloader-cli --fast --wait -u {{ editor }} -c editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloads basic editor - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project - - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} {% endfor -%} -{% endfor -%} \ No newline at end of file +{% endfor -%} From 46e042f8da25e4d70372f76b997857e1d611990b Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:14:05 -0400 Subject: [PATCH 09/27] Remove empty folder and use correct pvp argument --- .yamato/project-pack.yml | 2 +- testproject/Assets/Resources.meta | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100644 testproject/Assets/Resources.meta diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 46fde069b2..76dd4e847a 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -31,7 +31,7 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - upm-pvp pack --project {{ project.path }} --output upm-ci~/packages + - upm-pvp pack {{ project.path }} --output upm-ci~/packages dependencies: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors artifacts: diff --git a/testproject/Assets/Resources.meta b/testproject/Assets/Resources.meta deleted file mode 100644 index 52fd5df24a..0000000000 --- a/testproject/Assets/Resources.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 3f61bb7a9cfe67241ac932bf19d039c4 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: From 7dc97991306227360bee73db8c0498aa9897ccdb Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:17:22 -0400 Subject: [PATCH 10/27] See if standards check requires the editor --- .yamato/project-standards.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index aa68355e2a..f806f8b41f 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -44,9 +44,9 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: # .NET environment setup. Ensures required .NET SDK and formatting tools are available - dotnet --version - dotnet format --version - - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check - unity-downloader-cli --fast --wait -u {{ editor }} -c editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloads basic editor + - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project {% endfor -%} {% endfor -%} From 49b35d202d2fb85c873907918bb63c8cf827c3a8 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:26:16 -0400 Subject: [PATCH 11/27] package_pack runs faster than vetting_test or standards_check. Define it last so the other two are earlier in the VM queue --- .yamato/_run-all.yml | 6 ++++-- .yamato/_triggers.yml | 2 +- .yamato/project-standards.yml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 6ba0e948d0..aa24fd97f2 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -15,10 +15,12 @@ run_quick_checks: name: Run Quick Initial Checks dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_ubuntu - - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version - .yamato/vetting-test.yml#vetting_test + # ensure the code is running to our current standards + - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk + # ensure the package can be packed at at all on at least one platform + - .yamato/package-pack.yml#package_pack_-_ngo_win # Runs all package tests diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index 927a6c04bb..eb60f60292 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -51,8 +51,8 @@ pr_minimal_required_checks: name: Minimal PR checks dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_win - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk + - .yamato/package-pack.yml#package_pack_-_ngo_win triggers: expression: |- (pull_request.comment eq "ngo" OR diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index f806f8b41f..1e72086bf1 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -46,8 +46,8 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: - dotnet format --version - unity-downloader-cli --fast --wait -u {{ editor }} -c editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloads basic editor - - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project + - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} {% endfor -%} {% endfor -%} From 4731ff9bde5f9958d979ab0ebab587983bdd84dd Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:38:29 -0400 Subject: [PATCH 12/27] Put project_pack back to upm-ci --- .yamato/project-pack.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 76dd4e847a..a3360a826c 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -16,6 +16,7 @@ # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects #-------------------------------------------------------------------------------------- @@ -31,7 +32,8 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - - upm-pvp pack {{ project.path }} --output upm-ci~/packages + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it + - upm-ci project pack --project-path {{ project.path }} dependencies: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors artifacts: From 255a05cc831dd3696f9a1ad9cc16da631138d03b Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:41:59 -0400 Subject: [PATCH 13/27] Try running the vetting test on a smaller vm --- .yamato/vetting-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/vetting-test.yml b/.yamato/vetting-test.yml index d58f03e817..5d26deb39e 100644 --- a/.yamato/vetting-test.yml +++ b/.yamato/vetting-test.yml @@ -8,7 +8,7 @@ {% for editor in validation_editors.minimal -%} vetting_test: name: NGO - Vetting Test (Win, {{editor}} LTS) - agent: { type: Unity::VM, flavor: b1.large, image: package-ci/win11:v4 } + agent: { type: Unity::VM, flavor: b1.medium, image: package-ci/win11:v4 } commands: - npm install -g "upm-ci-utils@stable" --registry https://artifactory.prd.it.unity3d.com/artifactory/api/npm/upm-npm - unity-downloader-cli --fast --wait --unity-version {{ editor }} --components editor --arch x64 From 14ee396920b685c5ce97c456c14275833581fa47 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:44:09 -0400 Subject: [PATCH 14/27] move vetting_test out of quick_initial_checks --- .yamato/_run-all.yml | 2 -- .yamato/_triggers.yml | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index aa24fd97f2..bb01f50ffc 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -15,8 +15,6 @@ run_quick_checks: name: Run Quick Initial Checks dependencies: - # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version - - .yamato/vetting-test.yml#vetting_test # ensure the code is running to our current standards - .yamato/project-standards.yml#standards_ubuntu_testproject_trunk # ensure the package can be packed at at all on at least one platform diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index eb60f60292..a243a2b4e7 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -72,6 +72,9 @@ pr_code_changes_checks: name: Code changes PR checks # Run the following tests on a selection of different desktop platforms dependencies: + # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version + - .yamato/vetting-test.yml#vetting_test + # Run package EditMode and Playmode package tests on trunk and an older supported editor (6000.0) - .yamato/package-tests.yml#package_test_-_ngo_trunk_mac - .yamato/package-tests.yml#package_test_-_ngo_6000.0_win From 2390cbc663aa329bb3c9223cb534de7706699acb Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 20:47:13 -0400 Subject: [PATCH 15/27] Add run_quick_checks to desktop_standalone dependencies --- .yamato/desktop-standalone-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 11f3e916a1..83675a9616 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -53,6 +53,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ paths: - "artifacts/**/*" dependencies: + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} {% endfor -%} From 40c759d9ecaa5f4cc10c7ebdae473e619bd59e95 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 21:09:17 -0400 Subject: [PATCH 16/27] I think project_pack does nothing?? --- .yamato/desktop-standalone-tests.yml | 3 ++- .yamato/project-tests.yml | 3 ++- .yamato/project-updated-dependencies-test.yml | 15 ++++++++------- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 83675a9616..576a15462b 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -54,7 +54,8 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - "artifacts/**/*" dependencies: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 549b83d2cd..85623f6f98 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -44,7 +44,8 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: paths: - "test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endif -%} diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index 19ae953e27..d0a5e03f4c 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -4,7 +4,7 @@ # DESCRIPTION-------------------------------------------------------------------------- # This job is responsible fo validating package compatibility with latest dependency versions. # This helps detect potential breaking changes from dependency updates early - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all projects (testproject, minimalproject, testproject-tools-integration). @@ -15,17 +15,17 @@ # This job requires successful project packaging before execution (job dependency) # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) # The results are being generated in upm-ci~/test-results directory (specific of upm-ci) - + # QUALITY CONSIDERATIONS--------------------------------------------------------------------- # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp # To see where this job is included (in trigger job definitions) look into _triggers.yml file - -#-------------------------------------------------------------------------------------- - + +#-------------------------------------------------------------------------------------- + {% for project in projects.all -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} -updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: +updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: name : Updated Dependencies Test - NGO {{ project.name }} [{{ platform.name }}, {{ editor }}] agent: type: {{ platform.type }} @@ -42,7 +42,8 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: paths: - "upm-ci~/test-results/**/*" dependencies: - - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} From 7b2d2c8391b40e0743110ac8293905dc98e6b639 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 21:34:22 -0400 Subject: [PATCH 17/27] Remove --clean-library from any test runs --- .yamato/cmb-service-standalone-tests.yml | 2 +- .yamato/console-standalone-test.yml | 2 +- .yamato/desktop-standalone-tests.yml | 2 +- .yamato/mobile-standalone-test.yml | 4 ++-- .yamato/package-tests.yml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.yamato/cmb-service-standalone-tests.yml b/.yamato/cmb-service-standalone-tests.yml index bfca8d2fd9..b8b11ea86a 100644 --- a/.yamato/cmb-service-standalone-tests.yml +++ b/.yamato/cmb-service-standalone-tests.yml @@ -57,7 +57,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }} - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 98eecd9a7b..21d25c1ea8 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -90,7 +90,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library --timeout=1800 + - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 576a15462b..1174e97b43 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -81,7 +81,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 0a8f625702..a4d4baae01 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -97,11 +97,11 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --timeout=3600 {% else %} # Run tests for non-Android devices - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% {% endif %} artifacts: logs: diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 8080c2e156..1b2aade3b0 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -47,7 +47,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 --clean-library + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 artifacts: logs: paths: From 37aff393b8e2da2f372021f427f73e878a710698 Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 21:54:01 -0400 Subject: [PATCH 18/27] try swapping reruncount to retry --- .yamato/project-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 85623f6f98..04052b5258 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -38,13 +38,14 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --reruncount=1 --clean-library --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --clean-library --timeout=1800 artifacts: logs: paths: - "test-results/**/*" dependencies: # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} + # - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} From 454cf58587f694da155064a12af6aab9e93dbe5a Mon Sep 17 00:00:00 2001 From: Emma Date: Fri, 10 Oct 2025 22:52:43 -0400 Subject: [PATCH 19/27] Remove --clean-library from project_tests --- .yamato/project-tests.yml | 2 +- com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs | 2 ++ .../NetworkSceneManagerEventNotifications.cs | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 04052b5258..abf8d45d30 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -38,7 +38,7 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --clean-library --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 artifacts: logs: paths: diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs index 97e98f28f6..0e94185e7c 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs @@ -56,6 +56,8 @@ public IEnumerator ValidateUrlsAreValid() { Assert.IsTrue(tasks[i].Result, $"HelpUrls.{names[i]} has an invalid path! Path: {allUrls[i]}"); } + + Assert.Fail("Test reruns!"); } private async Task AreUnityDocsAvailableAt(string url) diff --git a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs index dba9897ab5..ddccd7c029 100644 --- a/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs +++ b/testproject/Assets/Tests/Runtime/NetworkSceneManager/NetworkSceneManagerEventNotifications.cs @@ -311,7 +311,6 @@ public IEnumerator SceneLoadingAndNotifications([Values] LoadSceneMode loadScene // Check error status for trying to load an invalid scene name LogAssert.Expect(LogType.Error, $"Scene '{k_InvalidSceneName}' couldn't be loaded because it has not been added to the build settings scenes in build list."); Assert.AreEqual(authority.SceneManager.LoadScene(k_InvalidSceneName, LoadSceneMode.Additive), SceneEventProgressStatus.InvalidSceneName); - Assert.Fail("Test reruns!"); } From c0ff854b134f02d0344ce5a6a76d142930987353 Mon Sep 17 00:00:00 2001 From: Emma Date: Sat, 11 Oct 2025 01:43:20 -0400 Subject: [PATCH 20/27] Separate editor and playmode tests --- .yamato/project-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index abf8d45d30..0ff0b1b2fe 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -38,7 +38,8 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 + - utr --testproject={{ project.path }} --suite=editor --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 + - utr --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 artifacts: logs: paths: From acbebff0eb9366cf982eb81c4f5a5557c5bb5cd8 Mon Sep 17 00:00:00 2001 From: Emma Date: Sat, 11 Oct 2025 02:02:33 -0400 Subject: [PATCH 21/27] try something else --- .yamato/package-tests.yml | 2 +- .yamato/project-tests.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 1b2aade3b0..c0ab1da1f8 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -47,7 +47,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --reruncount=1 + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 artifacts: logs: paths: diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 0ff0b1b2fe..68c39a82ae 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -38,8 +38,8 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - utr --testproject={{ project.path }} --suite=editor --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 - - utr --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 artifacts: logs: paths: From 753aa65108529d531854af0af4c608a96961be51 Mon Sep 17 00:00:00 2001 From: Emma Date: Sat, 11 Oct 2025 05:15:20 -0400 Subject: [PATCH 22/27] Use the working retry syntax --- .yamato/cmb-service-standalone-tests.yml | 2 +- .yamato/desktop-standalone-tests.yml | 2 +- .yamato/package-tests.yml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.yamato/cmb-service-standalone-tests.yml b/.yamato/cmb-service-standalone-tests.yml index b8b11ea86a..effa8c0412 100644 --- a/.yamato/cmb-service-standalone-tests.yml +++ b/.yamato/cmb-service-standalone-tests.yml @@ -57,7 +57,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }} - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 1174e97b43..02930fe9df 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -81,7 +81,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index c0ab1da1f8..f732690bd0 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -47,7 +47,8 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 + - UnifiedTestRunner --suite=editor --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 + - UnifiedTestRunner --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 artifacts: logs: paths: From 56098dfb2794e8b07f6e928035725081f5eb1a1d Mon Sep 17 00:00:00 2001 From: Emma Date: Sat, 11 Oct 2025 17:08:40 -0400 Subject: [PATCH 23/27] Remove runtime tests from project_tests and package_tests to test the standalone and cmb retry functionality --- .yamato/package-tests.yml | 2 +- .yamato/project-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index f732690bd0..696e33d212 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -48,7 +48,7 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - UnifiedTestRunner --suite=editor --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 - - UnifiedTestRunner --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 + # - UnifiedTestRunner --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 artifacts: logs: paths: diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 68c39a82ae..9337631222 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -39,7 +39,7 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 - - UnifiedTestRunner --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 + # - UnifiedTestRunner --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 artifacts: logs: paths: From 6b06a12e7244b1c56fd8dc5545a353c229f0d396 Mon Sep 17 00:00:00 2001 From: Emma Date: Mon, 13 Oct 2025 09:55:49 -0400 Subject: [PATCH 24/27] Try --clean-library-on-rerun --- .yamato/package-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 696e33d212..ebe2baacd8 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -47,8 +47,8 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - UnifiedTestRunner --suite=editor --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 - # - UnifiedTestRunner --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 + - UnifiedTestRunner --suite=editor --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 --clean-library-on-rerun + - UnifiedTestRunner --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --rerun-strategy=Test --retry=1 --clean-library-on-rerun artifacts: logs: paths: From 8703ac6dafd2a884accf70c083e16918ac2b5acc Mon Sep 17 00:00:00 2001 From: Emma Date: Mon, 13 Oct 2025 10:31:34 -0400 Subject: [PATCH 25/27] Put back --clean-library-on-rerun --- .yamato/cmb-service-standalone-tests.yml | 2 +- .yamato/code-coverage.yml | 2 +- .yamato/console-standalone-test.yml | 4 ++-- .yamato/desktop-standalone-tests.yml | 5 ++--- .yamato/mobile-standalone-test.yml | 6 +++--- .yamato/performance-tests.yml | 2 +- .yamato/project-standards.yml | 12 ++++++------ .yamato/project-tests.yml | 6 +++--- 8 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.yamato/cmb-service-standalone-tests.yml b/.yamato/cmb-service-standalone-tests.yml index effa8c0412..b01cf7040f 100644 --- a/.yamato/cmb-service-standalone-tests.yml +++ b/.yamato/cmb-service-standalone-tests.yml @@ -57,7 +57,7 @@ cmb_service_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }} - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 1715395118..98159dc0d4 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -39,7 +39,7 @@ code_coverage_{{ platform.name }}_{{ editor }}: commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library --artifacts-path=test-results + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --rerun-strategy=Test --retry=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 21d25c1ea8..6860b80e55 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -51,7 +51,7 @@ console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library --build-only --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --clean-library-on-rerun --build-only --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -90,7 +90,7 @@ console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} - - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --timeout=1800 + - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index 02930fe9df..ef9f30e81d 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -44,7 +44,7 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: paths: @@ -54,7 +54,6 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ - "artifacts/**/*" dependencies: - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} @@ -81,7 +80,7 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --timeout=1800 + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index a4d4baae01..99d3623c34 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -55,7 +55,7 @@ mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: UNITY_HANDLEUIINTERRUPTIONS: 1 commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library --build-only --timeout=1800 + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: paths: @@ -97,11 +97,11 @@ mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=3600 {% else %} # Run tests for non-Android devices - - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=3600 --device-id=%BOKKEN_DEVICE_ID% {% endif %} artifacts: logs: diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index ff06ff2ba7..d2ff40e166 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -36,7 +36,7 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin {% endif %} commands: - unity-downloader-cli -u {{ editor }} -c Editor --wait {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor - - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library --dontreportperformancedata + - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --rerun-strategy=Test --retry=1 --clean-library-on-rerun --dontreportperformancedata # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: logs: diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 1e72086bf1..6a70daf149 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -8,24 +8,24 @@ # Project structure validation # Coding convention adherence # Solution file synchronization - + # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs configurations are generated using nested loops through: # 1. For all NGO projects (testproject, testproject-tools-interation, minimalproject) # 2. For default platform only (Ubuntu) since standards would not vary between platforms (no need for checks on more platforms) # 3. For default editor version (trunk) since standards would not vary between editors (no need for checks on more editors) - + # TECHNICAL CONSTRAINTS--------------------------------------------------------------- # Requires .NET SDK installed (should be preinstalled on the image so we just check for version) # Needs Unity Editor for solution synchronization # Uses custom standards validation tool (netcode.standards) # Generates no test artifacts (pass/fail only). Eventual failure will be visible in the logs - + # QUALITY THOUGHTS-------------------------------------------------------------------- # While testproject validation would be sufficient, since it validates both project and package (where package is our main concern) jobs for all projects are being generated to ensure that we conform to quality standards in all projects. # TODO: consider modifying the approach and adding checks for minimal supported editor. In case of NGOv1.X it has proven to yield different results (But since NGOv2.X support starts from 6000.0 editor it's not that time pressuring) # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + #------------------------------------------------------------------------------------ {% for project in projects.all -%} @@ -44,10 +44,10 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: # .NET environment setup. Ensures required .NET SDK and formatting tools are available - dotnet --version - dotnet format --version - + - unity-downloader-cli --fast --wait -u {{ editor }} -c editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloads basic editor - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} {% endfor -%} -{% endfor -%} +{% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index 9337631222..dd6948dce3 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -38,15 +38,15 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 - # - UnifiedTestRunner --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800 + - UnifiedTestRunner --testproject={{ project.path }} --suite=playmode --artifacts-path=test-results --editor-location=.Editor --rerun-strategy=Test --retry=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: - "test-results/**/*" dependencies: # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} - # - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} From fccced3560a4c888541d8bcb01fe5c5f144ce579 Mon Sep 17 00:00:00 2001 From: Emma Date: Mon, 13 Oct 2025 10:32:04 -0400 Subject: [PATCH 26/27] completely delete project-pack --- .yamato/README.md | 5 +-- .yamato/project-pack.yml | 44 ------------------- .yamato/project-updated-dependencies-test.yml | 1 - 3 files changed, 2 insertions(+), 48 deletions(-) delete mode 100644 .yamato/project-pack.yml diff --git a/.yamato/README.md b/.yamato/README.md index 2e0a761714..7db6767555 100644 --- a/.yamato/README.md +++ b/.yamato/README.md @@ -1,7 +1,7 @@ # Netcode for GameObjects CI Documentation ## Overview -This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. +This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. Specifics of each test are described within related files (for example .yamato/package-tests.yml) and this file present high level overview related to overall implementation. ## Test Configurations @@ -9,7 +9,6 @@ CI related files are present inside .yamato/ folder and we can distinguish speci ### Helper jobs - `.yamato/package-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. -- `.yamato/project-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. This packs all packages of a given project. - `.yamato/_run-all.yml` responsible for grouping tests into groups for easier management (for example "all console tests"). - `.yamato/_triggers.yml` responsible for defining triggers (PR, nightly, weekly etc.) and defining which tests to run. - `disable-burst-if-requested.py` responsible for helping to disable burst if needed. @@ -77,4 +76,4 @@ Currently, the CI implementation supports the following platforms: ## Design Considerations In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. -The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. \ No newline at end of file +The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml deleted file mode 100644 index a3360a826c..0000000000 --- a/.yamato/project-pack.yml +++ /dev/null @@ -1,44 +0,0 @@ -{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. ---- - -# DESCRIPTION-------------------------------------------------------------------------- - # This job is responsible for packing a specific project. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. - # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. - -# CONFIGURATION STRUCTURE-------------------------------------------------------------- - # Jobs configurations are generated using nested loops through: - # 1. For all projects (testproject, minimalproject, testproject-tools-integration). - # 2. For all desktop platforms (Win, Ubuntu, Mac) - -# TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- - # Job does not require Unity Editor in order to perform packing. - # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. - -# QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # To see where this job is included (in trigger job definitions) look into _triggers.yml file - # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects - -#-------------------------------------------------------------------------------------- - -{% for project in projects.all -%} -{% for platform in test_platforms.desktop -%} -project_pack_-_{{ project.name }}_{{ platform.name }}: - name: Project Pack - {{ project.name }} [{{ platform.name }}] - agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.smaller_flavor }} -{% if platform.model %} - model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) -{% endif %} - commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - - upm-ci project pack --project-path {{ project.path }} - dependencies: - - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors - artifacts: - packages: - paths: - - "upm-ci~/packages/**/*" -{% endfor -%} -{% endfor -%} diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index d0a5e03f4c..7d749ce18a 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -42,7 +42,6 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: paths: - "upm-ci~/test-results/**/*" dependencies: - # - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} From 11aa2460342528dd5adba1a277d6483ec93362da Mon Sep 17 00:00:00 2001 From: Emma Date: Mon, 13 Oct 2025 10:32:57 -0400 Subject: [PATCH 27/27] Revert failure in HelpUrlTests --- com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs index 0e94185e7c..97e98f28f6 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/HelpUrlTests.cs @@ -56,8 +56,6 @@ public IEnumerator ValidateUrlsAreValid() { Assert.IsTrue(tasks[i].Result, $"HelpUrls.{names[i]} has an invalid path! Path: {allUrls[i]}"); } - - Assert.Fail("Test reruns!"); } private async Task AreUnityDocsAvailableAt(string url)