From 0c74940ab52c716a2fc248ff1b646f3390e94fa1 Mon Sep 17 00:00:00 2001 From: Stefan Schubert Date: Mon, 21 Oct 2024 16:39:57 +0200 Subject: [PATCH 1/5] first instability test --- .yamato/config.metadata | 5 ++++- .yamato/upm-ci.yml | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.yamato/config.metadata b/.yamato/config.metadata index 98b30ee34a..fe17d1de53 100644 --- a/.yamato/config.metadata +++ b/.yamato/config.metadata @@ -72,4 +72,7 @@ ios_and_tvos_macos_bokken_image: package-ci/macos-13:v4.50.0 test_category: - name: performance - - name: all \ No newline at end of file + - name: all + +instabilies_install_win: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.bat --output run_standalone_instability_detection-latest.bat --retry 5 || exit 0 +instabilies_run_win: run_standalone_instability_detection-latest.bat 0.1.6 || exit 0 \ No newline at end of file diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 514ba0c92a..68b61ccae8 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -34,6 +34,9 @@ - move /Y .\Packages\com.unity.inputsystem\Samples.meta .\Assets # Now run our full test suite that sits in Assets/Tests by running UTR on our project. - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilies_install_win }} + - {{ instabilies_run_win }} artifacts: UTR_Output.zip: paths: From 3e3e48b9c032abfff6e974ee08194a50618e2de8 Mon Sep 17 00:00:00 2001 From: Stefan Schubert Date: Mon, 21 Oct 2024 17:43:43 +0200 Subject: [PATCH 2/5] version update for instability tool --- .yamato/config.metadata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.yamato/config.metadata b/.yamato/config.metadata index fe17d1de53..96201fa5ca 100644 --- a/.yamato/config.metadata +++ b/.yamato/config.metadata @@ -75,4 +75,4 @@ test_category: - name: all instabilies_install_win: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.bat --output run_standalone_instability_detection-latest.bat --retry 5 || exit 0 -instabilies_run_win: run_standalone_instability_detection-latest.bat 0.1.6 || exit 0 \ No newline at end of file +instabilies_run_win: run_standalone_instability_detection-latest.bat 0.5.1 || exit 0 \ No newline at end of file From 1f846cb794530515dba4bccb265b3b2cc51b3032 Mon Sep 17 00:00:00 2001 From: Stefan Schubert Date: Wed, 23 Oct 2024 15:22:09 +0200 Subject: [PATCH 3/5] add instabilities runner to all test jobs --- .yamato/config.metadata | 12 ++++++++++-- .yamato/upm-ci.yml | 25 +++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.yamato/config.metadata b/.yamato/config.metadata index 96201fa5ca..e6f97bcb31 100644 --- a/.yamato/config.metadata +++ b/.yamato/config.metadata @@ -36,15 +36,18 @@ platforms_nix: type: Unity::VM::osx image: package-ci/macos-13:default flavor: m1.mac + instabilities_run: sh ./run_standalone_instability_detection-latest.sh macos 0.5.1 || exit 0 - name: mac_standalone type: Unity::VM::osx image: package-ci/macos-13:default flavor: m1.mac + instabilities_run: sh ./run_standalone_instability_detection-latest.sh macos 0.5.1 || exit 0 runtime: StandaloneOSX - name: mac_standalone_il2cpp type: Unity::VM::osx image: package-ci/macos-13:default flavor: m1.mac + instabilities_run: sh ./run_standalone_instability_detection-latest.sh macos 0.5.1 || exit 0 runtime: StandaloneOSX scripting-backend: Il2Cpp installscript: unity-downloader-cli -c editor -c StandaloneSupport-IL2CPP --wait --fast -u @@ -52,15 +55,18 @@ platforms_nix: type: Unity::VM image: package-ci/ubuntu-20.04:v4.50.0 flavor: b1.large + instabilities_run: sh ./run_standalone_instability_detection-latest.sh ubuntu 0.5.1 || exit 0 - name: linux_standalone type: Unity::VM image: package-ci/ubuntu-20.04:v4.50.0 flavor: b1.large + instabilities_run: sh ./run_standalone_instability_detection-latest.sh ubuntu 0.5.1 || exit 0 runtime: StandaloneLinux64 - name: linux_standalone_il2cpp type: Unity::VM image: package-ci/ubuntu-20.04:v4.50.0 flavor: b1.large + instabilities_run: sh ./run_standalone_instability_detection-latest.sh ubuntu 0.5.1 || exit 0 runtime: StandaloneLinux64 scripting-backend: Il2Cpp installscript: unity-downloader-cli -c editor -c StandaloneSupport-IL2CPP --wait --fast -u @@ -74,5 +80,7 @@ test_category: - name: performance - name: all -instabilies_install_win: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.bat --output run_standalone_instability_detection-latest.bat --retry 5 || exit 0 -instabilies_run_win: run_standalone_instability_detection-latest.bat 0.5.1 || exit 0 \ No newline at end of file +instabilities_install_win: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.bat --output run_standalone_instability_detection-latest.bat --retry 5 || exit 0 +instabilities_run_win: run_standalone_instability_detection-latest.bat 0.5.1 || exit 0 + +instabilities_install_nix: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.sh --output run_standalone_instability_detection-latest.sh --retry 5 || exit 0 \ No newline at end of file diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 68b61ccae8..66cf0695de 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -35,8 +35,8 @@ # Now run our full test suite that sits in Assets/Tests by running UTR on our project. - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "win" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} --report-performance-data --performance-project-id=InputSystem after: - - {{ instabilies_install_win }} - - {{ instabilies_run_win }} + - {{ instabilities_install_win }} + - {{ instabilities_run_win }} artifacts: UTR_Output.zip: paths: @@ -75,6 +75,9 @@ - mv ./Packages/com.unity.inputsystem/Samples.meta ./Assets # Now run our full test suite that sits in Assets/Tests by running UTR on our project. - ./utr --testproject . --timeout=1200 --editor-location=.Editor --artifacts_path=upm-ci~/test-results/isolation-com.unity.inputsystem.tests --suite=playmode {% if platform.name == "mac" %} --suite=editor {% endif %} {% if platform.name == "linux" %} --suite=editor {% endif %} {% if category.name == "performance" %} --category=Performance {% endif %} --api-profile=NET_4_6 {% if platform.runtime %} --platform {{ platform.runtime }} {% endif %} {% if platform.scripting-backend %} --scripting-backend {{ platform.scripting-backend }} {% endif %} --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ platform.instabilities_run }} artifacts: UTR_Output.zip: paths: @@ -96,6 +99,9 @@ build_ios_{{ editor.version }}_{{ category.name }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c iOS -u {{ editor.version }} --fast --wait - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ platform.instabilities_run }} artifacts: players: paths: @@ -117,6 +123,9 @@ run_ios_{{ editor.version }}_{{ category.name }}: commands: - {{ utr_install_nix }} - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ platform.instabilities_run }} artifacts: logs: paths: @@ -134,6 +143,9 @@ build_tvos_{{ editor.version }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c AppleTV -u {{ editor.version }} --fast --wait - ./utr --suite=playmode --platform=tvOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ platform.instabilities_run }} artifacts: players: paths: @@ -154,6 +166,9 @@ run_tvos_{{ editor.version }}: commands: - {{ utr_install_nix }} - ./utr --suite=playmode --platform=tvOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ platform.instabilities_run }} artifacts: logs: paths: @@ -172,6 +187,9 @@ build_android_{{ editor.version }}_{{ backend.name }}_{{ category.name }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c Android -u {{ editor.version }} --fast --wait - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=Android --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --scripting-backend={{ backend.name }} --build-only --repository --performance-project-id=InputSystem + after: + - {{ instabilities_install_win }} + - {{ instabilities_run_win }} artifacts: players: paths: @@ -205,6 +223,9 @@ run_android_{{ editor.version }}_{{ backend.name }}_{{ category.name }}: - start %ANDROID_SDK_ROOT%\platform-tools\adb.exe connect %BOKKEN_DEVICE_IP% - if not exist build\test-results mkdir build\test-results - powershell %ANDROID_SDK_ROOT%\platform-tools\adb.exe logcat -d > build/test-results/device_log.txt + after: + - {{ instabilities_install_win }} + - {{ instabilities_run_win }} # Set uploadable artifact paths artifacts: logs: From a83fe90b3e95e05da04528d7144b17dfc9c43de9 Mon Sep 17 00:00:00 2001 From: Stefan Schubert Date: Wed, 23 Oct 2024 16:45:09 +0200 Subject: [PATCH 4/5] revert some afters --- .yamato/upm-ci.yml | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 66cf0695de..83f6e2520e 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -99,9 +99,6 @@ build_ios_{{ editor.version }}_{{ category.name }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c iOS -u {{ editor.version }} --fast --wait - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only --report-performance-data --performance-project-id=InputSystem - after: - - {{ instabilities_install_nix }} - - {{ platform.instabilities_run }} artifacts: players: paths: @@ -122,10 +119,7 @@ run_ios_{{ editor.version }}_{{ category.name }}: - .yamato/upm-ci.yml#build_ios_{{ editor.version }}_{{ category.name }} commands: - {{ utr_install_nix }} - - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem - after: - - {{ instabilities_install_nix }} - - {{ platform.instabilities_run }} + - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem artifacts: logs: paths: @@ -143,9 +137,6 @@ build_tvos_{{ editor.version }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c AppleTV -u {{ editor.version }} --fast --wait - ./utr --suite=playmode --platform=tvOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only --report-performance-data --performance-project-id=InputSystem - after: - - {{ instabilities_install_nix }} - - {{ platform.instabilities_run }} artifacts: players: paths: @@ -166,9 +157,6 @@ run_tvos_{{ editor.version }}: commands: - {{ utr_install_nix }} - ./utr --suite=playmode --platform=tvOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem - after: - - {{ instabilities_install_nix }} - - {{ platform.instabilities_run }} artifacts: logs: paths: @@ -222,8 +210,7 @@ run_android_{{ editor.version }}_{{ backend.name }}_{{ category.name }}: after: - start %ANDROID_SDK_ROOT%\platform-tools\adb.exe connect %BOKKEN_DEVICE_IP% - if not exist build\test-results mkdir build\test-results - - powershell %ANDROID_SDK_ROOT%\platform-tools\adb.exe logcat -d > build/test-results/device_log.txt - after: + - powershell %ANDROID_SDK_ROOT%\platform-tools\adb.exe logcat -d > build/test-results/device_log.txt - {{ instabilities_install_win }} - {{ instabilities_run_win }} # Set uploadable artifact paths From 054ab1729f5c1dd9f3063aeda46c0f98c2a8763d Mon Sep 17 00:00:00 2001 From: Stefan Schubert Date: Wed, 23 Oct 2024 17:06:01 +0200 Subject: [PATCH 5/5] 2nd try for ios/tvos --- .yamato/config.metadata | 3 ++- .yamato/upm-ci.yml | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.yamato/config.metadata b/.yamato/config.metadata index e6f97bcb31..fb0b5355cf 100644 --- a/.yamato/config.metadata +++ b/.yamato/config.metadata @@ -83,4 +83,5 @@ test_category: instabilities_install_win: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.bat --output run_standalone_instability_detection-latest.bat --retry 5 || exit 0 instabilities_run_win: run_standalone_instability_detection-latest.bat 0.5.1 || exit 0 -instabilities_install_nix: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.sh --output run_standalone_instability_detection-latest.sh --retry 5 || exit 0 \ No newline at end of file +instabilities_install_nix: curl -s https://artifactory.prd.it.unity3d.com/artifactory/automation-and-tooling/infrastructure-instability-detection/standalone/setup/run_standalone_instability_detection-latest.sh --output run_standalone_instability_detection-latest.sh --retry 5 || exit 0 +instabilities_run_mac: sh ./run_standalone_instability_detection-latest.sh macos 0.5.1 || exit 0 \ No newline at end of file diff --git a/.yamato/upm-ci.yml b/.yamato/upm-ci.yml index 83f6e2520e..0e703a7226 100644 --- a/.yamato/upm-ci.yml +++ b/.yamato/upm-ci.yml @@ -99,6 +99,9 @@ build_ios_{{ editor.version }}_{{ category.name }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c iOS -u {{ editor.version }} --fast --wait - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ instabilities_run_mac }} artifacts: players: paths: @@ -120,6 +123,9 @@ run_ios_{{ editor.version }}_{{ category.name }}: commands: - {{ utr_install_nix }} - ./utr --suite=playmode {% if category.name == "performance" %} --category=Performance {% endif %} --platform=iOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ instabilities_run_mac }} artifacts: logs: paths: @@ -137,6 +143,9 @@ build_tvos_{{ editor.version }}: - {{ unity_downloader_install }} - unity-downloader-cli -c Editor -c AppleTV -u {{ editor.version }} --fast --wait - ./utr --suite=playmode --platform=tvOS --editor-location=.Editor --testproject=. --player-save-path=build/players --artifacts_path=build/logs --build-only --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ instabilities_run_mac }} artifacts: players: paths: @@ -157,6 +166,9 @@ run_tvos_{{ editor.version }}: commands: - {{ utr_install_nix }} - ./utr --suite=playmode --platform=tvOS --player-load-path=build/players --artifacts_path=build/test-results --report-performance-data --performance-project-id=InputSystem + after: + - {{ instabilities_install_nix }} + - {{ instabilities_run_mac }} artifacts: logs: paths: