diff --git a/.yamato/README.md b/.yamato/README.md new file mode 100644 index 0000000000..f5eafa856b --- /dev/null +++ b/.yamato/README.md @@ -0,0 +1,80 @@ +# Netcode for GameObjects CI Documentation + +## Overview +This document provides an overview of the Continuous Integration (CI) implementation for Netcode for GameObjects. +Specifics of each test are described within related files (for example .yamato/package-tests.yml) and this file present high level overview related to overall implementation. + +## Test Configurations +CI related files are present inside .yamato/ folder and we can distinguish specific tests + +### Helper jobs +- `.yamato/package-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. +- `.yamato/project-pack.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. This packs all packages of a given project. +- `.yamato/_run-all.yml` responsible for grouping tests into groups for easier management (for example "all console tests"). +- `.yamato/_triggers.yml` responsible for defining triggers (PR, nightly, weekly etc.) and defining which tests to run. +- `disable-burst-if-requested.py` responsible for helping to disable burst if needed. +- `.yamato/project.metafile` being the central configuration file defining test environments, platforms, projects, editors and other parameters used. + +### In-Editor Tests +- `.yamato/package-tests.yml` responsible for running package tests. +- `.yamato/project-tests.yml` responsible for running project tests. +- `.yamato/project-updated-dependencies-test.yml` responsible for running tests after updating dependencies to latest ones. +- `.yamato/project-standards.yml` responsible for running standards check on a package. +- `.yamato/code-coverage.yml` responsible for running code coverage analysis. +- `.yamato/performance-tests.yml` responsible for running performance tests. +- `.yamato/webgl-build.yml` responsible for making a build for webgl platform. + +### Standalone Tests +- `.yamato/desktop-standalone-tests.yml` responsible for running tests on standalone desktops. +- `.yamato/console-standalone-test.yml` responsible for running tests on standalone consoles. +- `.yamato/mobile-standalone-test.yml` responsible for running tests on standalone mobiles. + +### Wrench jobs +This file is auto generated by wrench and used for automating release process. +Those tests can also be used by our CI as dependencies. +- `.yamato/wrench/api-validation-jobs.yml` responsible for validating api for a given package version (for example if there are no breaking changes when updating patch version) +- `.yamato/wrench/package-pack-jobs.yml` responsible for generating package artifacts (.tgz) required for testing and publishing. +- `.yamato/wrench/preview-a-p-v.yml` responsible for validating PVP (Package Verification Profiles) standards for the package. +- `.yamato/wrench/promotion-jobs.yml` responsible for publishing. +- `.yamato/wrench/publish-trigger.yml` responsible for defining jobs that need to pass in order to publish. +- `.yamato/wrench/recipe-regeneration.yml` responsible for verifying regeneration of wrench recipes (no manual changes). +- `.yamato/wrench/validation-jobs.yml` responsible for running package tests. +- `.yamato/wrench/wrench_config.json` this is the configuration file for wrench. +- `.yamato/generated-scripts/` is a collection of infrastructure instability detection tools. + +## Jobs Configuration Structure +This section explains some configuration details that may be confusing while looking at .yml files. + +### Job Generation +Our configuration uses a dynamic job generation approach based on the `.yamato/project.metafile` configuration. While it may appear that only one job is defined per file, the system (Yamato) actually generates multiple jobs through nested loops. +Usually only 1 job is defined and then generated with multiple configurations/parameters but in case of standalone tests 2 jobs are defined which are responsible for building and then testing. + +### Job Naming Convention +Because the jobs are being generated their names will be filled by given parameters from project.metfaile. For example jobs from .yamato/console-standalone-test may have names like: +- Display Name (in Yamato): `Build testproject - [win, 6000.0, il2cpp]` +- Internal Job Name (used when adding dependency for example): `console_standalone_build_testproject_win_6000.0` + +## Platform Support +Currently, the CI implementation supports the following platforms: +1. Console Platforms: + - Switch + - PS4 + - PS5 + - Xbox360 + - XboxOne +2. Mobile Platforms: + - Android + - iOS +3. Desktop Platforms: + - Windows + - Ubuntu + - macOS +4. Unity Editor Versions + - Supports NGOv1.X (2021.3+ editors) +5. Architectures + - x64 + - ARM64 (This is present for consoles/mobiles but will be extended. More information is present in specific standalone test files) + +## Design Considerations +In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations. +The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements. \ No newline at end of file diff --git a/.yamato/_run-all.yml b/.yamato/_run-all.yml index 011a93208d..372152bb19 100644 --- a/.yamato/_run-all.yml +++ b/.yamato/_run-all.yml @@ -1,6 +1,16 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- +# DESCRIPTION-------------------------------------------------------------------------- + # Those jobs group together related test as dependencies to allow to easily manage running a given set of tests. + # This enables efficient test execution for different validation scenarios + +# QUALITY CONSIDERATIONS--------------------------------------------------------------- + # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. + +#----------------------------------------------------------------------------------- + + # Runs all package tests run_all_package_tests: name: Run All Package Tests @@ -130,13 +140,7 @@ run_all_project_tests_desktop_standalone: {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} {% for backend in scripting_backends -%} -# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since -# 1) The 2021 editor will soon be out of support -# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks -# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on -{% if editor != "2021.3" or platform.name != "ubuntu" %} - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }} -{% endif -%} {% endfor -%} {% endfor -%} {% endfor -%} @@ -164,13 +168,7 @@ run_all_project_tests_desktop_standalone_2021: {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for backend in scripting_backends -%} -# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since -# 1) The 2021 editor will soon be out of support -# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks -# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on -{% if platform.name != "ubuntu" %} - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_2021.3 -{% endif -%} {% endfor -%} {% endfor -%} {% endfor -%} @@ -182,7 +180,7 @@ run_all_project_tests_mobile_standalone: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.all -%} - - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -195,7 +193,7 @@ run_all_project_tests_mobile_standalone_trunk: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.default -%} - - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -206,7 +204,7 @@ run_all_project_tests_mobile_standalone_2021: dependencies: {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} - - .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_2021.3 + - .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_{{ platform.name }}_2021.3 {% endfor -%} {% endfor -%} @@ -218,10 +216,7 @@ run_all_project_tests_console_standalone: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.all -%} -# When building consoles with 2023 editor there are errors related to SDK version or uninstalled components. I will disable those for now and will add to my investigation list -{% if editor != "2023.3" %} - - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} -{% endif -%} + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -234,7 +229,7 @@ run_all_project_tests_console_standalone_trunk: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.default -%} - - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }} + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} @@ -245,6 +240,6 @@ run_all_project_tests_console_standalone_2021: dependencies: {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} - - .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_2021.3 + - .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_{{ platform.name }}_2021.3 {% endfor -%} {% endfor -%} diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index e8e7a330ce..e3e42504f4 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -1,6 +1,43 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- +# DESCRIPTION-------------------------------------------------------------------------- + # This configuration defines three main CI trigger patterns: + # 1. Pull Request Validation: Validation performed on PR basis + # 2. Nightly Development: Test set run nightly (validates most important test cases) + # 3. Weekly Full Validation: Test set run weekly (validates all test cases to prevent any surprises) + # Each pattern represents different balance between validation depth, execution time and CI resource usage + +# TRIGGER PATTERNS------------------------------------------------------------------- + # Pull Request: + # This test validates Standards, Package tests, Project tests and Desktop standalone tests to ensure that main platforms are covered + # Triggers on PRs to develop, develop, and release branches + # Focuses on critical validation paths that we should validate before merging PRs + # Cancels previous runs on new commits + # Excludes draft PRs + + # Nightly: + # This test validates same subset as pull_request_trigger with addition of mobile/console tests and webgl builds + # Runs daily on develop (local configuration) + # Includes all test types but only on trunk. + # Adds platform-specific and APV validation + + # Weekly: + # This test validates same subset as develop_nightly but runs per all supported editors as well as executes code coverage test and runs project standards per project + # Runs across all supported editor versions + # Includes code coverage analysis + # Validates all projects and standards + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated by ensuring that all dependencies are successful. + # The dependencies are taken from _run-all.yml file where we can gather multiple tests into proper sets + +# QUALITY CONSIDERATIONS--------------------------------------------------------------- + # It's important to ensure that all dependencies exist (this can be verified in Yamato) since a modification in parameters may result in a given job not being generated, and thus we will not be able to run such erroneous job. + + +#----------------------------------------------------------------------------------- + # Run all relevant tasks when a pull request targeting the develop or release branch is created or updated. pull_request_trigger: name: Pull Request Trigger (develop, develop-2.0.0, & release branches) @@ -66,6 +103,9 @@ develop_nightly: # Build player for webgl platform on trunk and 2021 editors - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_ubuntu_trunk - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_2021.3 + # Clean import test + - .yamato/clean-import-job.yml#clean_import_testproject_trunk + - .yamato/clean-import-job.yml#clean_import_testproject_2021.3 # Run all tests on weekly bases diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index c0aa7e59fd..fcc64a4ccf 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -1,10 +1,30 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Runs package tests in order to determine code coverage of the NGO package. -# In essence it's performing the same task as .yamato/package-test jobs with the overhead being the measured code coverage -# It's ok for code coverage to be performed only on one platform (default) since code coverage won't change much between those. -# Default platform was chosen (ubuntu) since it's the fastest and most resource friendly with default editor. +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for executing package tests with code coverage analysis enabled. + # Coverage analysis provides insights into: + # Test coverage metrics for NGO assemblies + # Line and branch coverage statistics + # Generated HTML reports for coverage visualization + # Additional metrics for coverage analysis + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) + # 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors) + +#TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # In theory this job also runs package tests, but we don't want to use it as default since is heavier (because of added coverage analysis) and coverage is not changing that often + # Requires Unity Editor installation + # Burst compilation is disabled to ensure accurate coverage measurement + # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise, coverage results will not be visible + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: somewhere in 2025 we will be able to upload resuls to CodeCov from public repos + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + + {% for platform in test_platforms.default -%} {% for editor in validation_editors.default -%} code_coverage_{{ platform.name }}_{{ editor }}: @@ -13,16 +33,18 @@ code_coverage_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - - DISPLAY=:0 upm-ci package test -u {{ editor }} --package-path com.unity.netcode.gameobjects --enable-code-coverage --code-coverage-options 'generateAdditionalMetrics;generateHtmlReport;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime' --extra-utr-arg="--extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun" + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models + - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: - - "upm-ci~/test-results/**/*" - - "upm-ci~/test-results/CoverageResults/**/*" + - "test-results/**/*" dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }}_upmCI + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/console-standalone-test.yml b/.yamato/console-standalone-test.yml index 09723ab7fb..0bbd9f9a00 100644 --- a/.yamato/console-standalone-test.yml +++ b/.yamato/console-standalone-test.yml @@ -1,37 +1,57 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Builds a player on console standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build. -# Default project (testptoject) in this case is used as a context. -# Builds/Tests are made on each console platform (PS4, PS5, Switch, XboxOne, XboxSeriesX) as in project.metafile declaration -# Builds/Tests are made on each supported editor as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for Console platform test validation. + # Those tests cover both PlayMode and EditMode tests from package test assemblies. -# For SOME of the console devices it's necessary to split build and run phases so it was split for all -# For all consoles we need to use il2cpp scripting backend (so no testing with mono) -# Switch works only with ARM64 and the rest with x64 architectures -# For now all platforms used for building are windows based +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: + # 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) + # 2. For all supported Unity Editor versions (for NGOv1.X this means 2021.3+ editors) + # 3. For the default project. -# Builds are made with x64 architecture machines since those are compatible to run on ARM64 devices +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # For console devices a split is required into two phases: + # 1. Build Phase: Creates standalone players for console platforms + # 2. Run Phase: Executes runtime tests on actual console devices + # The Run phase uses build job as dependency + + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + +# PLATFORM SPECIFICS----------------------------------------------------------------- + # Common Requirements: + # All consoles require IL2CPP scripting backend + # Platform-specific SDK environment variables + + # Platform-Specific: + # Switch: ARM64 architecture only + # Other Consoles: x64 architecture + # Each console requires specific SDK paths and tools + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # TODO: consider adding all projects that have tests + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + + + +# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_build -%} {% for editor in validation_editors.all -%} -# When building consoles with 2023 editor there are errors related to SDK version or uninstalled components. I will disable those for now and will add to my investigation list -{% if editor != "2023.3" %} -build_{{ project.name }}_{{ platform.name }}_{{ editor }}: +console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp{% if platform.name == "switch" %}, arm64{% endif %}] agent: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - # Installing tools. unity-downloader-cli and utr should be already preinstalled on the image - - sudo pip install unity-downloader-cli - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat - - # Platform specific Build - - utr --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 - + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} + - UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -49,37 +69,28 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}: logs: paths: - "artifacts/**/*" -{% endif -%} {% endfor -%} {% endfor -%} {% endfor -%} -# Executes RuntimeTests of the NGO package in the Standalone build for consoles +# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.console_test -%} {% for editor in validation_editors.all -%} -# When building consoles with 2023 editor there are errors related to SDK version or uninstalled components. I will disable those for now and will add to my investigation list -{% if editor != "2023.3" %} -run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: +console_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} {% if platform.model %} - model: {{ platform.model }} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} commands: - # Installing tools. - - sudo pip install unity-downloader-cli - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat - - # Platform specific Execution - - utr --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 - + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} + - UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 variables: # PS4 related SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00' @@ -95,8 +106,7 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: paths: - "test-results/**/*" dependencies: - - .yamato/console-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }} -{% endif -%} + - .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/desktop-standalone-tests.yml b/.yamato/desktop-standalone-tests.yml index eceff41b3c..ec17d182fc 100644 --- a/.yamato/desktop-standalone-tests.yml +++ b/.yamato/desktop-standalone-tests.yml @@ -1,57 +1,50 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Builds a player on desktop standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build. -# Default project (testptoject) in this case is used as a context. -# Builds are made on each desktop platform as in project.metafile declaration -# Builds are made on each supported editor as in project.metafile declaration -# Builds are made with different scripting backends as in project.metafile declaration -# ARM64 architectures are for now not considered since Windows_arm64 is recommended to use only after builds (would require separation here) and when it comes to macOS_arm64 there is problem with OpenCL not being available +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for Desktop platform test validation. + # Those tests cover both PlayMode and EditMode tests from package test assemblies. -# Build phase +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency: + # 1. For all desktop platform (Windows, macOS, Ubuntu) + # 2. For all supported Unity Editor versions (for NGOv1.X this means 2021.3+ editors) + # 3. For the default project. + # 4. For all scripting backends (mono, il2cpp) + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # For desktop devices a split is into two phases is not required but we use it for consistency with setup of others standalone platforms: + # 1. Build Phase: Creates standalone players for desktop platforms + # 2. Run Phase: Executes runtime tests on actual desktop devices + # The Run phase uses build job as dependency + + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # TODO: consider adding all projects that have tests + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + +#----------------------------------------------------------------------------------- + + +# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} {% for backend in scripting_backends -%} -# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since -# 1) The 2021 editor will soon be out of support -# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks -# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on -{% if editor != "2021.3" or platform.name != "ubuntu" %} desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}: name : Standalone Build - NGO {{ project.name }} - [{{ platform.name }}, {{ editor }}, {{ backend }}] agent: type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} - commands: -# Installing tools -{% if platform.name == "ubuntu" %} - - sudo apt-get update -q - - sudo apt install -qy imagemagick -{% endif %} - - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - -# Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - -# Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait - -# Build Player - - | -{% if platform.name == "win" %} - utr.bat --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% else %} - ./utr --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% endif %} - + commands: + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --testproject={{ project.path }} --scripting-backend={{ backend }} --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --artifacts-path=artifacts --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: paths: @@ -59,10 +52,8 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ logs: paths: - "artifacts/**/*" - dependencies: - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} -{% endif %} {% endfor -%} {% endfor -%} {% endfor -%} @@ -71,49 +62,29 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ -# Run phase +# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} {% for backend in scripting_backends -%} -# There is an error 'No usable version of libssl' when it comes to building with combination of ubuntu and 2021 editor. It's not really worth to address it since -# 1) The 2021 editor will soon be out of support -# 2) We still have the same coverage by using mac and ubuntu is used in a whole bunch of different tests, so I'm quite sure it won't lead to any bug leaks -# 3) The solution could be to manually install libss1.1 (since image has libssl3 and this causes conflicts on 2021 editor related to dotnet versions and support). There were several threads on this topic which can be looked on -{% if editor != "2021.3" or platform.name != "ubuntu" %} desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }}: name : Standalone Test - NGO {{ project.name }} - [{{ platform.name }}, {{ editor }}, {{ backend }}] agent: type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} image: {{ platform.image }} flavor: {{ platform.flavor }} - commands: -# Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - - unity-downloader-cli -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} --fast --wait - -# Run Standalone tests - - | -{% if platform.name == "win" %} - utr.bat --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 -{% else %} - ./utr --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 -{% endif %} - + commands: + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} + - UnifiedTestRunner --suite=playmode --player-load-path=build/players --artifacts-path=test-results --testproject={{ project.path }} --editor-location=.Editor --playergraphicsapi=Null --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800 artifacts: logs: paths: - "test-results/**/*" dependencies: - .yamato/desktop-standalone-tests.yml#desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{ editor }} -{% endif %} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/mobile-standalone-test.yml b/.yamato/mobile-standalone-test.yml index 2ca2444b86..8a1ca65b4d 100644 --- a/.yamato/mobile-standalone-test.yml +++ b/.yamato/mobile-standalone-test.yml @@ -1,41 +1,56 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- - -# Builds a player on mobile standalone platform and executes RuntimeTests (equivalent to PlayMode tests) of the NGO package in the Standalone build. -# Default project (testptoject) in this case is used as a context. -# Builds/Tests are made on each mobile platform (Android and iOS) as in project.metafile declaration -# Builds/Tests are made on each supported editor as in project.metafile declaration - -# For mobile devices it's necessary to split build and run phases -# For iOS we need to use il2cpp. For android we could use both but il2cpp is recommended so for now we will only use il2cpp as scripting backend -# iOS works only with ARM64 and Android is tested with both ARM64 and ARMv7 -# Builds are made with x64 architecture machines since those are compatible to run on ARM64 devices +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for Mobile platform test validation. + # Those tests cover both PlayMode and EditMode tests from package test assemblies. + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. For all mobile platform (Android, iOS) + # 2. For all supported Unity Editor versions (for NGOv1.X this means 2021.3+ editors) + # 3. For the default project. + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # For mobile devices a split is required into two phases: + # 1. Build Phase: Creates standalone players for mobile platforms + # 2. Run Phase: Executes runtime tests on actual mobile devices + # The Run phase uses build job as dependency + # Note: More of a Unity specific but test assemblies need to be included in the build phase command + # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices + +# PLATFORM SPECIFICS-------------------------------------------------------------------- + # iOS Requirements: + # Must use IL2CPP scripting backend + # Only supports ARM64 architecture + # Builds on macOS agents only + + # Android Requirements: + # Uses IL2CPP scripting backend (recommended over Mono) + # Supports both ARM64 and ARMv7 architectures + # Can be built on any desktop platform + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: consider adding all projects that have tests + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + + +# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_build -%} {% for editor in validation_editors.all -%} -build_{{ project.name }}_{{ platform.name }}_{{ editor }}: - name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp] +mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: + name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} - commands: - # Installing tools - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --fast --wait - - # Platform specific Build -{% if platform.base == "win" %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat - - utr.bat --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% else %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr - - chmod +x ./utr - - ./utr --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} + commands: + - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --wait + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --testproject={{ project.path }} --architecture={{ platform.architecture }} --scripting-backend=il2cpp --editor-location=.Editor --artifacts-path=artifacts --testfilter="Unity.Netcode.RuntimeTests.*" --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: players: paths: @@ -43,64 +58,52 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}: logs: paths: - "artifacts/**/*" - - variables: - CI: true - ENABLE_BURST_COMPILATION: False {% endfor -%} {% endfor -%} {% endfor -%} -# Executes RuntimeTests of the NGO package in the Standalone build for mobiles +# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------ {% for project in projects.default -%} {% for platform in test_platforms.mobile_test -%} {% for editor in validation_editors.all -%} -run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}: +mobile_standalone_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp] agent: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} {% if platform.model %} - model: {{ platform.model }} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - # Skip repository cloning - skip_checkout: true commands: - # Installing tools - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - # Installing editor. We still need the editor to run tests on standalone build - - unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} --fast --wait - + # Installing editor. We still need the editor to run tests on standalone build and for that the Editor is required + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c il2cpp {% if platform.base == "mac" %} -c ios {% else %} -c android {% endif %} + {% if platform.standalone == "Android" %} # Download standalone UnityTestRunner and ADB setup - - command: curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey.pub -O %USERPROFILE%/.android/adbkey.pub - command: wget http://artifactory-slo.bf.unity3d.com/artifactory/mobile-generic/android/ADBKeys.zip!/adbkey -O %USERPROFILE%/.android/adbkey - command: gsudo NetSh Advfirewall set allprofiles state off - + # Connect to Android device - command: '"%ANDROID_SDK_ROOT%\platform-tools\adb.exe" connect %BOKKEN_DEVICE_IP%' - - # Run tests + + # Run tests for Android devices - | set ANDROID_DEVICE_CONNECTION=%BOKKEN_DEVICE_IP% - utr --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --player-connection-ip=%BOKKEN_HOST_IP% --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 {% else %} - # Download standalone UnityTestRunner - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr - - # Run tests - - ./utr --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 + + # Run tests for non-Android devices + - UnifiedTestRunner --suite=playmode --platform={{ platform.standalone }} --artifacts-path=test-results --player-load-path=build/players --testproject={{ project.path }} --editor-location=.Editor --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=3600 {% endif %} artifacts: logs: paths: - "test-results/**/*" dependencies: - - .yamato/mobile-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }} + - .yamato/mobile-standalone-test.yml#mobile_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/package-pack.yml b/.yamato/package-pack.yml index 1d54e47212..61e794f110 100644 --- a/.yamato/package-pack.yml +++ b/.yamato/package-pack.yml @@ -1,9 +1,28 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Packs Netcode for GameObjects together with performing initial checks. -# For this job no specific platform support and no running Unity instance is required so small agent (as per project.metafile definition) could be used to save resources and speed up the process. -# If everyone adheres to this rule it can create bottlenecks (since everyone would use this machine) so we decided to pack with the same platform as the given job runs on +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for packing a specific package. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. + # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. + # The job performs additional validation by using Package Verification Pipeline (PVP). It includes x-ray validation for quick package structure verification + # Because of that validation we can detect errors at the early stage of testing so not to waste CI resources + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. For all desktop platforms (Windows, Ubuntu, macOS) + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # Job does not require Unity Editor in order to perform packing. + # Job uses PVP x-ray for lightweight validation + # Job generates both packages artifacts and pvp-results file. + # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: we should aim to replace target PVP profile from supported to gold + +#------------------------------------------------------------------------------------ + {% for platform in test_platforms.desktop -%} package_pack_-_ngo_{{ platform.name }}: name: Package Pack (and x-ray) - NGO [{{ platform.name }}] @@ -11,6 +30,9 @@ package_pack_-_ngo_{{ platform.name }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} timeout: 0.25 variables: XRAY_PROFILE: "supported ./pvpExceptions.json" @@ -25,27 +47,4 @@ package_pack_-_ngo_{{ platform.name }}: packages: paths: - "upm-ci~/**" -{% endfor -%} - -# This is in essence the same job as the one above with the difference that upm-ci is used instead of upm-pvp -# The reason for using it is that I had some problems with Code Coverage which in its current form uses upm-ci but if we would use the other pack job (the one above) we would need to use upm-pvp -# I had some problems with getting it to work so as temporary solution I created this pack job which is used ONLY as a dependency of Code Coverage job (other jobs use the above definition of pack job) -# TODO: remove this job and utilize the above one for Code Coverage job. This is tracked in MTT-11383 -{% for platform in test_platforms.default -%} -{% for project in projects.default -%} -package_pack_-_ngo_{{ platform.name }}_upmCI: - name: Package Pack (legacy upm-ci) - NGO [{{ platform.name }}] - agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} - timeout: 0.25 - commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - upm-ci project pack --project-path {{ project.path }} - artifacts: - packages: - paths: - - "upm-ci~/packages/**/*" -{% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/package-tests.yml b/.yamato/package-tests.yml index 5373551ca2..0d354967ae 100644 --- a/.yamato/package-tests.yml +++ b/.yamato/package-tests.yml @@ -1,9 +1,29 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Executes PlayMode and EditMode tests of the NGO package in the Editor context -# Those tests run in the editor so we don't need to consider different scripting backends or architectures -# Tests are executed for all supported editors on each desktop platform as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for execution of package-specific tests in Unity Editor context + # Those tests cover both PlayMode and EditMode tests from package test assemblies + # Additionally it combines Package Verification Pipeline (PVP) validation. This ensures that package is compatible with Unity standards + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs are generated using nested loops through: + # 1. For all desktop platforms (Windows, Ubuntu, macOS) + # 2. For all supported Unity Editor versions (for NGOv1.X this means 2021.3+ editors) + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # This job runs in Editor context only (no player builds required) + # No scripting backend variations needed (Editor context) + # Architecture variations not applicable (Editor context) + # Requires project packaging as prerequisite (dependency job) + # Uses PVP for package validation. Specifically it looks for supported profiles which we should conform to but takes ./pvpExceptions.json file into consideration where we note our known issues related to PVP checks + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: we should aim to replace target PVP profile from supported to gold + +#------------------------------------------------------------------------------------ + {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} package_test_-_ngo_{{ editor }}_{{ platform.name }}: @@ -12,25 +32,22 @@ package_test_-_ngo_{{ editor }}_{{ platform.name }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} variables: XRAY_PROFILE: "supported ./pvpExceptions.json" UNITY_EXT_LOGGING: 1 commands: - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - # Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr -{% endif %} - - # Validate packages. + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. + + # Validate PVP checks for package. - upm-pvp test --unity .Editor --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --results pvp-results - upm-pvp require {% if platform.name == "win" %}"%XRAY_PROFILE%"{% else %}"$XRAY_PROFILE"{% endif %} --results pvp-results + # Run UTR to test packages. - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --filter "com.unity.netcode.gameobjects" --unity .Editor - - {% if platform.name == "ubuntu" %}DISPLAY=:0 {% endif %} {% if platform.name == "win" %} utr.bat {% else %} ./utr {% endif %} --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --extra-editor-arg=-testCategory --extra-editor-arg=!Performance --reruncount=1 --clean-library-on-rerun + - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --artifacts-path=test-results "--ff={ops.upmpvpevidence.enable=true}" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun artifacts: logs: paths: diff --git a/.yamato/performance-tests.yml b/.yamato/performance-tests.yml index abdbef444e..e18b2ff19d 100644 --- a/.yamato/performance-tests.yml +++ b/.yamato/performance-tests.yml @@ -1,8 +1,27 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Performance tests for the `com.unity.netcode.gameobjects` package. No performance data will be reported. -# Performance tests are executed within editor on desktop platforms in context of default project (testproject) +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for executing performance tests for NGO package. + # Its configuration is set to not report any data but just to give results (at least in current state since we don't have any tests to run). + # Currently, because of lack of performance tests this job will always return "no tests have been selected" and because oif that it's not included in any trigger jobs. + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated using nested loops through: + # 1. For all desktop platforms (Windows, Ubuntu, macOS) + # 2. For all supported Unity Editor versions (For NGOv1.X it means 2021.3+) + # 3. For the default project (project is used only as a context for the build). TODO-comment: if performance tests would be included in projects then we should make an appropriate split. + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # Tests are run in Editor context only + # No performance metrics are reported to monitoring systems + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: Currently NGO don't have any performance tests so this job is a placeholder for the future. We should discuss how to approach the topic of performance testing + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + +#------------------------------------------------------------------------------------ + {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} {% for project in projects.default -%} @@ -12,44 +31,17 @@ performance_editor_tests_-_NGO_{{ platform.name }}_{{ editor }}_no_data_reportin type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} - commands: - # Installing tools -{% if platform.name == "ubuntu" %} - - sudo apt-get update -q - - sudo apt install -qy imagemagick -{% endif %} - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - # Platform specific UTR setup -{% if platform.name == "win" %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} - - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - - # Build Player - - | -{% if platform.name == "win" %} - utr.bat --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata -{% else %} - ./utr --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata -{% endif %} - + commands: + - unity-downloader-cli -u {{ editor }} -c Editor --wait {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor + - UnifiedTestRunner --suite=editor --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --timeout=3600 --artifacts-path=artifacts --extra-editor-arg=-assemblyNames --extra-editor-arg=Unity.NetCode.* --extra-editor-arg=-testCategory --extra-editor-arg=Performance --extra-editor-arg=-enablePackageManagerTraces --reruncount=1 --clean-library-on-rerun --dontreportperformancedata + # TODO: when performance tests will be present we need to add actuall execution of this test artifacts: logs: paths: - - '*.log' - - '*.xml' - - artifacts/**/* - - {{ project.name }}/Logs/** - - {{ project.name }}/Library/*.log - - {{ project.name }}/*.log - - {{ project.name }}/Builds/*.log - - build/test-results/** + - "artifacts/**/*" {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-pack.yml b/.yamato/project-pack.yml index 0963d99271..d9dd6143d5 100644 --- a/.yamato/project-pack.yml +++ b/.yamato/project-pack.yml @@ -1,9 +1,26 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Jobs that iterate through and packs all NGO projects listed (to use in different jobs) -# For this job no specific platform support and no running Unity instance is required so small agent (as per project.metafile definition) coul be used to save resources and speed up the process -# If everyone adheres to this rule it can create bottlenecks (since everyone would use this machine) so we decided to pack project with the same platform as the given job runs on +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for packing a specific project. It generates package artifacts (.tgz) required for testing and publishing, ensuring all dependencies are properly bundled and validated before any test execution. + # The job itself doesn't test anything specific but rather it prepares project packages that will be consumed by other pipeline jobs. + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated using nested loops through: + # 1. For all projects (testproject, minimalproject, testproject-tools-integration). + # 2. For all desktop platforms (Win, Ubuntu, Mac) + +# TECHNICAL CONSIDERATIONS-------------------------------------------------------------------- + # Job does not require Unity Editor in order to perform packing. + # In theory, we could just use one platform for packing projects (for example ubuntu) but in order to reduce confusion we are using same platform as the job utilizing project pack as dependency. + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp. Additionally this would allow us to run PVP checks on projects + + +#-------------------------------------------------------------------------------------- + {% for project in projects.all -%} {% for platform in test_platforms.desktop -%} project_pack_-_{{ project.name }}_{{ platform.name }}: @@ -12,8 +29,11 @@ project_pack_-_{{ project.name }}_{{ platform.name }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - - npm install -g upm-ci-utils@stable --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - upm-ci project pack --project-path {{ project.path }} artifacts: packages: diff --git a/.yamato/project-standards.yml b/.yamato/project-standards.yml index 0bb442971e..dd88c60e68 100644 --- a/.yamato/project-standards.yml +++ b/.yamato/project-standards.yml @@ -1,9 +1,32 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Project standards are being checked for package (in project context). -# It should be enough to perform the test on default project (testproject) but to make sure that all projects conform to our standards job for each of them will be available. -# Tests are executed for default editor (trunk) on default platform (ubuntu) as in project.metafile declaration since results would be the same across editors and platforms. +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for validating project compliance with NGO coding standards and conventions + # Standards validation includes: + # Code formatting compliance + # Project structure validation + # Coding convention adherence + # Solution file synchronization + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated using nested loops through: + # 1. For all NGO projects (testproject, testproject-tools-interation, minimalproject) + # 2. For default platform only (Ubuntu) since standards would not vary between platforms (no need for checks on more platforms) + # 3. For default editor version (trunk) since standards would not vary between editors (no need for checks on more editors) + +# TECHNICAL CONSTRAINTS--------------------------------------------------------------- + # Requires .NET SDK installed (should be preinstalled on the image so we just check for version) + # Needs Unity Editor for solution synchronization + # Uses custom standards validation tool (netcode.standards) + # Generates no test artifacts (pass/fail only). Eventual failure will be visible in the logs + +# QUALITY THOUGHTS-------------------------------------------------------------------- + # While testproject validation would be sufficient, since it validates both project and package (where package is our main concern) jobs for all projects are being generated to ensure that we conform to quality standards in all projects. + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + +#------------------------------------------------------------------------------------ + {% for project in projects.all -%} {% for platform in test_platforms.default -%} {% for editor in validation_editors.all -%} @@ -13,18 +36,18 @@ standards_{{ platform.name }}_{{ project.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - # .NET environment setup + # .NET environment setup. Ensures required .NET SDK and formatting tools are available - dotnet --version - dotnet format --version - - # Unity setup - - pip install unity-downloader-cli --upgrade --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - unity-downloader-cli -u {{ editor }} -c editor --wait --fast - - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit - - # Standards check - - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check + + - unity-downloader-cli --fast --wait -u {{ editor }} -c editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloads basic editor + + - .Editor/Unity -batchmode -nographics -logFile - -executeMethod Packages.Rider.Editor.RiderScriptEditor.SyncSolution -projectPath {{ project.path }} -quit # This command is used to invoke Unity in a "headless" mode. It's used to sync the project + - dotnet run --project=dotnet-tools/netcode.standards -- --project={{ project.path }} --check # Runs standards check {% endfor -%} {% endfor -%} {% endfor -%} \ No newline at end of file diff --git a/.yamato/project-tests.yml b/.yamato/project-tests.yml index ec91e04540..153f452170 100644 --- a/.yamato/project-tests.yml +++ b/.yamato/project-tests.yml @@ -1,10 +1,29 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Executes PlayMode and EditMode tests of the given project (ones that have tests) in the Editor context -# Those tests don't include NGO package tests since they are handled on their respective jobs. -# Those tests run in the editor so we don't need to consider different scripting backends or architectures -# Tests are executed for all supported editors on each desktop platform as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # This job executes project-specific tests in Unity Editor context + # Those tests cover both PlayMode and EditMode tests from project test assemblies + # NGO package tests are NOT being executed within this job (those are handled in separate package test jobs) + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated using nested loops through: + # 1. For all projects WITH TESTS (filtered by has_tests flag) (testproject, testproject-tools-interation) [For more info look into project.metafile configuration] + # 2. For all desktop platforms (Windows, Ubuntu, macOS) + # 3. For all supported Unity Editor versions (for NGOv1.X this means 2021.3+ editors) + +# TECHNICAL CONSIDERATIONS--------------------------------------------------------------- + # This job runs in Editor context only (no player builds is required) + # No scripting backend variations needed (Editor context) + # Architecture variations not applicable (Editor context) + # Requires project packaging as prerequisite (dependency job) + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + +#------------------------------------------------------------------------------------ + {% for project in projects.all -%} {% if project.has_tests == "true" -%} {% for platform in test_platforms.desktop -%} @@ -15,10 +34,13 @@ test_{{ project.name }}_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm - - unity-downloader-cli -u {{ editor }} -c Editor --fast --wait - - {% if platform.name == "ubuntu" %}DISPLAY=:0 && {% endif %}upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--extra-editor-arg=-testCategory --extra-editor-arg=!Performance --reruncount=1 --clean-library-on-rerun" + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Installing basic editor for tests execution + - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type project-tests --extra-utr-arg="--reruncount=1 --clean-library-on-rerun" # project tests execution via upm-ci artifacts: logs: paths: diff --git a/.yamato/project-updated-dependencies-test.yml b/.yamato/project-updated-dependencies-test.yml index cdc44c9e8e..5b7530ac60 100644 --- a/.yamato/project-updated-dependencies-test.yml +++ b/.yamato/project-updated-dependencies-test.yml @@ -1,9 +1,28 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Executes package tests for each package referenced in the project using the newest compatible version of the package dependencies. -# This is just to ensure that no critical change is made. It will be run on a default editor and default platform (it can be discussed if we should run it per all editors and platforms) -{% for project in projects.default -%} +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible fo validating package compatibility with latest dependency versions. + # This helps detect potential breaking changes from dependency updates early + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated using nested loops through: + # 1. For all projects (testproject, minimalproject, testproject-tools-integration). + # 2. For all desktop platforms (Win, Ubuntu, Mac) + # 3. For all supported editors (For NGOv1.X it means 2021.3+) + +# TECHNICAL CONSIDERATIONS---------------------------------------------------------------- + # This job requires successful project packaging before execution (job dependency) + # This job tests only NGO package dependencies (com.unity.netcode.gameobjects) + # The results are being generated in upm-ci~/test-results directory (specific of upm-ci) + +# QUALITY CONSIDERATIONS--------------------------------------------------------------------- + # TODO: Currently upm-ci is being used but in the future it will be replaced by upm-pvp + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + +#-------------------------------------------------------------------------------------- + +{% for project in projects.all -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: @@ -12,16 +31,18 @@ updated-dependencies_{{ project.name }}_NGO_{{ platform.name }}_{{ editor }}: type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} commands: - - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm + - npm install upm-ci-utils@stable -g --registry https://artifactory.prd.cds.internal.unity3d.com/artifactory/api/npm/upm-npm # upm-ci is not preinstalled on the image so we need to download it - upm-ci project test -u {{ editor }} --project-path {{ project.path }} --type updated-dependencies-tests --package-filter com.unity.netcode.gameobjects artifacts: logs: paths: - "upm-ci~/test-results/**/*" dependencies: - - path: .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} - rerun: always + - .yamato/project-pack.yml#project_pack_-_{{ project.name }}_{{ platform.name }} {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/.yamato/project.metafile b/.yamato/project.metafile index e924f8426c..0c9c6ae1e1 100644 --- a/.yamato/project.metafile +++ b/.yamato/project.metafile @@ -1,4 +1,22 @@ -# The small agent was created to handle jobs that don't involve running Unity (for example pack job). +# DESCRIPTION----------------------------------------------------------------------------------------------------------------- + # This file is the central configuration file defining test environments, platforms, projects, editors and other parameters used. + # Data from this file is used to generate CI pipeline configurations through templating in other .yamato/ files. + # This provides consistent environment definitions across all CI jobs. It's easier to modify values in central place (for example image reference) than having to modify every job + +# VALUE MEANING---------------------------------------------------------------------------------------------------------------------------- + # Usual fields being defined are + # name --> Identifier used to distinguish different configurations in the CI pipeline + # type --> Specifies the Bokken agent type (e.g., Unity::VM, Unity::VM::osx, Unity::mobile::shield) + # image --> Defines the package-ci Bokken image to use for the environment (e.g., package-ci/ubuntu-22.04:v4). This is basically a device configuration + # flavor --> Determines the VM size/resources (e.g., b1.small, b1.large, m1.mac) + # standalone --> Specifies the build target platform (e.g., StandaloneLinux64, Android, IOS) + # model --> Defines specific hardware model requirements (e.g., rtx2080, SE for iPhone SE) + # base --> Indicates the base operating system for build operations (e.g., win, mac) + # architecture --> Specifies the target CPU architecture (e.g., armv7, arm64) + + +# SMALL AGENT CONFIGURATION--------------------------------------------------------------------------------------------------- +# The small agent was created to handle jobs that don't involve running Unity and are in general super light when it comes to resource usage (for example pack job). # It uses ubuntu since Linux VMs are faster and cheaper to provision than Mac or Windows Virtual Machines (VMs). # The b1.small flavour is nearly always sufficient for jobs that don’t involve running Unity Editor. small_agent_platform: @@ -7,7 +25,13 @@ small_agent_platform: image: package-ci/ubuntu-22.04:v4 flavor: b1.small -# Platforms that project will/can be tested on. + +# PLATFORM CONFIGURATIONS---------------------------------------------------------------------------------------------------------------- +# test_platforms specifies platforms that project will/can be tested on. +# For desktops we include Windows, Ubuntu and Mac, for mobiles we include Android and iOS, for consoles we include ps4, ps5, switch, xbox360 and xboxOne +# For mobile and console a split was required for build and test jobs and this is also applied for desktops for consistency (though for desktop this split is not required) + + test_platforms: # Default platform is used for all basic jobs. Ubuntu was chosen since it's fastest and most available default: @@ -31,10 +55,12 @@ test_platforms: model: rtx2080 - name: mac type: Unity::VM::osx - image: package-ci/macos-13:v4 + image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below) flavor: m1.mac standalone: StandaloneOSX + model: M1 # The default model (an x64 Intel Mac VM) quite often caused a known issue of doing all the bitflips in packages resulting in failures # For mobile devices there is a split between the build and run phase so there is a need of splitting specification for both + # TODO: For android we could consider adding ARM64 configuration mobile_build: - name: android type: Unity::VM @@ -43,7 +69,7 @@ test_platforms: standalone: Android base: win architecture: armv7 - # iOS modern builds are ARM64-only, thus no testing with armv7 (as in android case) + # iOS modern builds are ARM64-only, thus no testing with armv7 - name: ios-arm64 type: Unity::VM::osx image: package-ci/macos-13:v4 @@ -121,30 +147,33 @@ test_platforms: flavor: b1.large standalone: GameCoreScarlett - -# Editors to be used for testing. -# Since NGOv2 official support started from U6 it means that only those editors should be used for testing +# EDITOR CONFIGURATIONS------------------------------------------------------------------------------- +# Editors to be used for testing. NGOv1.X official support started from 2021.3 editor +# TODO: When a new editor will be released it should be added to this list. Correct also if we will stop supporting one of the editors present here + validation_editors: default: - trunk all: - 2021.3 - 2022.3 - - 2023.3 - 6000.0 - 6000.1 - trunk -# Scripting backends used by Standalone RunTimeTests +# Scripting backends used by Standalone RunTimeTests--------------------------------------------------- + scripting_backends: - mono - il2cpp -# Projects within the repository that will be tested. Name will be used -# for job ids, so it should not contain spaces/non-supported characters -# The default project will also be used for validation, code coverage etc. +# PROJECTS CONFIGURATIONS------------------------------------------------------------------------------- +# Projects within the repository that will be tested/build. +# The default project should be used for general validation, code coverage and other tests where choice of project should not matter (because we are interested in NGO package) +# has_tests --> describes if projects contains any tests to run. + projects: default: - name: testproject diff --git a/.yamato/webgl-build.yml b/.yamato/webgl-build.yml index 97a182acdb..33fac79782 100644 --- a/.yamato/webgl-build.yml +++ b/.yamato/webgl-build.yml @@ -1,53 +1,55 @@ -{% metadata_file .yamato/project.metafile %} +{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file. --- -# Builds a player on WebGl standalone platform without executing any tests. -# This setup performs build-only validation since WebGL runs in browser and for tests to be executed we would need to -# consider having a web server, browser automation and overall complex test setup. -# Default project (testptoject) in this case is used as a context. -# WebGL requires Il2cpp scripting backend so mono is not considered -# ARM64 architectures are not considered since Windows_arm64 is recommended to use only after builds and when it comes to macOS_arm64 there is problem with OpenCL not being available -# Builds are made on each desktop platform as in project.metafile declaration +# DESCRIPTION-------------------------------------------------------------------------- + # This job is responsible for validating a successful building of a player on WebGl standalone platform. + # This job WILL NOT execute any tests, we only validate the building process. + # This is because WebGL runs in browser and for tests to be executed we would need to consider having a web server, browser automation and overall complex test setup which currently we don't have. + +# CONFIGURATION STRUCTURE-------------------------------------------------------------- + # Jobs configurations are generated using nested loops through: + # 1. For the default project (project is used only as a context for the build). + # 2. For all desktop platforms (Windows, Ubuntu, macOS) + # 3. For all supported Unity Editor versions (For NGOv1.X it means 2021.3+) + +# TECHNICAL CONSIDERATIONS---------------------------------------------------------------- + # WebGL requires IL2CPP scripting backend (Mono is not supported) + # We are not using ARM64 architectures since we only perform a build action. x64 architectures are preferred for build phase (in order to optimize available resource usage) + # We only perform build validation (no runtime testing) + +# QUALITY CONSIDERATIONS-------------------------------------------------------------------- + # In the future we could try to implement an infrastructure to run test in webgl context but this could be quite complicated and would need to be evaluated if it's worth it + # To see where this job is included (in trigger job definitions) look into _triggers.yml file + +#-------------------------------------------------------------------------------------- + {% for project in projects.default -%} {% for platform in test_platforms.desktop -%} {% for editor in validation_editors.all -%} webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: WebGl Build - {{ project.name }} [{{ platform.name }}, {{ editor }}, il2cpp] agent: - type: {% if platform.name == "mac" %} {{ platform.type }} {% else %} {{ platform.type }}::GPU {% endif %} + type: {{ platform.type }} image: {{ platform.image }} flavor: {{ platform.flavor }} - commands: - # Installing tools - - pip install unity-downloader-cli --index-url https://artifactory.prd.it.unity3d.com/artifactory/api/pypi/pypi/simple - - # Platform specific UTR setup - - | -{% if platform.name == "win" %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr.bat --output utr.bat -{% else %} - curl -s https://artifactory.prd.it.unity3d.com/artifactory/unity-tools-local/utr-standalone/utr --output utr && chmod +x utr +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) {% endif %} + commands: + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor -c webgl -c il2cpp {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models. Downloading the editor with additional webgl and il2cpp components - # Installing editor - - unity-downloader-cli -u {{ editor }} -c Editor -c webgl -c il2cpp -w --fast - - # Build Player - - | -{% if platform.name == "win" %} - utr.bat --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg="-cloudEnvironment staging" --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% else %} - ./utr --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg="-cloudEnvironment staging" --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 -{% endif %} - + # The following step builds the player with defined options such as: + # Suite parameter if defined since it's a mandatory field to define which test suite should be used, but it doesn't matter in this case since we won't run any tests (--suite) + # Editor is run in batchmode, which means that Unity runs command line arguments without the need for human interaction. It also suppresses pop-up windows that require human interaction (such as the Save Scene window). We should always run Unity in batch mode when using command line arguments, because it allows automation to run without interruption. (--extra-editor-arg=-batchmode) + # Engine is initialized in ‘nographics’ mode since we don't need any graphics for this case (--extra-editor-arg=-nographics) + # In case of failure the job will be rerunned once (--reruncount=1) with clean library (--clean-library-on-rerun) + # This will perform only building phase (--build-only) with a timeout of 3m (--timeout=1800) + - UnifiedTestRunner --suite=playmode --platform=WebGL --scripting-backend=il2cpp --testproject={{ project.path }} --editor-location=.Editor --artifacts_path=artifacts --player-save-path=build/players --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800 artifacts: logs: paths: - "artifacts/**/*" - "build/players/**/*" - variables: - CI: true - ENABLE_BURST_COMPILATION: False {% endfor -%} {% endfor -%} {% endfor -%} diff --git a/com.unity.netcode.gameobjects/Tests/Editor/Timing/NetworkTimeTests.cs b/com.unity.netcode.gameobjects/Tests/Editor/Timing/NetworkTimeTests.cs index be53a0df7f..9256951193 100644 --- a/com.unity.netcode.gameobjects/Tests/Editor/Timing/NetworkTimeTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Editor/Timing/NetworkTimeTests.cs @@ -3,6 +3,7 @@ using System.Linq; using NUnit.Framework; using UnityEngine; +using UnityEngine.TestTools; using Random = System.Random; namespace Unity.Netcode.EditorTests @@ -38,6 +39,7 @@ public void TestTimeAsFloat(double d, float f, uint tickRate) } [Test] + [UnityPlatform(exclude = new[] { RuntimePlatform.OSXPlayer, RuntimePlatform.OSXEditor })] // Tracked in MTT-11608 [TestCase(53.55d, 53.5d, 10u)] [TestCase(1013553.55d, 1013553.5d, 10u)] [TestCase(0d, 0d, 10u)] @@ -52,6 +54,7 @@ public void TestToFixedTime(double time, double expectedFixedTime, uint tickRate } [Test] + [UnityPlatform(exclude = new[] { RuntimePlatform.OSXPlayer, RuntimePlatform.OSXEditor })] // Tracked in MTT-11608 [TestCase(34d, 0)] [TestCase(17.32d, 0.2d / 60d)] [TestCase(-42.44d, 1d / 60d - 0.4d / 60d)] diff --git a/com.unity.netcode.gameobjects/Tests/Runtime/UniversalRpcTests.cs b/com.unity.netcode.gameobjects/Tests/Runtime/UniversalRpcTests.cs index 584a13ae62..69674c20d9 100644 --- a/com.unity.netcode.gameobjects/Tests/Runtime/UniversalRpcTests.cs +++ b/com.unity.netcode.gameobjects/Tests/Runtime/UniversalRpcTests.cs @@ -1668,6 +1668,7 @@ public void TestMutualRecursion() Clear(); } + serverObj.Stop = true; WaitForMessageReceivedWithTimeTravel(serverList); Assert.IsFalse(serverObj.Stop); diff --git a/pvpExceptions.json b/pvpExceptions.json index 4deefeeb86..b3aaff683e 100644 --- a/pvpExceptions.json +++ b/pvpExceptions.json @@ -5,6 +5,11 @@ "CHANGELOG.md: line 9: Unreleased section is not allowed for public release" ] }, + "PVP-300-4": { + "errors": [ + "Failed to find a suitable OpenCL device, baking cannot use GPU lightmapper." + ] + }, "PVP-150-1": { "errors": [ "Unity.Netcode.Components.NetworkTransform: void OnSynchronize(ref BufferSerializer): empty tag",