Add test failure reporting (#232)

Author: Jamie Brynes

.buildkite/hooks/post-checkout

@@ -7,6 +7,6 @@ if [[ -z "${BUILDKITE-}" ]]; then
   ci/bootstrap.sh
   exit 0
 fi
-if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE}" != "trigger-pipelines" ]]; then
+if [[ "${BUILDKITE_AGENT_META_DATA_QUEUE}" != "trigger-pipelines" ]] && [[ "${BUILDKITE_AGENT_META_DATA_OS}" == "windows" ]]; then
   ci/bootstrap.sh
 fi

.buildkite/hooks/pre-command

@@ -2,6 +2,6 @@
 set -euo pipefail
 [[ -n "${DEBUG:-}" ]] && set -x
 
-if [[ "${BUILDKITE_AGENT_META_DATA_CAPABLE_OF_BUILDING}" == "gdk-for-unity" ]]; then
+if [[ "${BUILDKITE_AGENT_META_DATA_CAPABLE_OF_BUILDING}" == "gdk-for-unity" ]] && [[ "${BUILDKITE_AGENT_META_DATA_OS}" == "windows" ]]; then
   spatial auth login --log_level debug
 fi

.buildkite/premerge.steps.yaml

@@ -31,16 +31,48 @@ common: &common
       - exit_status: 128
         limit: 3
 
+linux: &linux
+  agents:
+    - "capable_of_building=gdk-for-unity"
+    - "environment=production"
+    - "permission_set=builder"
+    - "platform=linux"  # if you need a different platform, configure this: macos|linux|windows.
+    - "queue=v3-1562766449-11b126f880607f28-------z"
+    - "scaler_version=2"
+  timeout_in_minutes: 60 # TODO(ENG-548): reduce timeout once agent-cold-start is optimised.
+  retry:
+    automatic:
+        # This is designed to trap and retry failures because agent lost connection. Agent exits with -1 in this case.
+      - exit_status: -1
+        limit: 3
+
 # NOTE: step labels turn into commit-status names like {org}/{repo}/{pipeline}/{step-label}, lower-case and hyphenated.
 # These are then relied on to have stable names by other things, so once named, please beware renaming has consequences.
 
+dag: true
+
 steps:
   - label: "test"
+    id: "test"
     command: bash -c ci/test.sh
     <<: *common
     artifact_paths:
       - logs/**/*
 
+  - label: "annotate test results :pencil2:"
+    depends_on: 
+      - step: "test"
+        allow_failure: true
+    plugins: 
+      - improbable/test-summary#d7289cac8297018fd1b452dcf828979307b3ebc6:
+          inputs:
+            - label: ":octagonal_sign: Test failures "
+              artifact_path: "logs\\nunit\\*.xml"
+              type: nunit
+          formatter:
+            type: details
+    <<: *linux
+
   - label: "build :android:"
     command: bash -c .shared-ci/scripts/build-worker.sh
     <<: *common

ci/test.sh

@@ -13,7 +13,7 @@ source .shared-ci/scripts/pinned-tools.sh
 PROJECT_DIR="$(pwd)"
 mkdir -p "${PROJECT_DIR}/logs/"
 
-EDITMODE_TEST_RESULTS_FILE="${PROJECT_DIR}/logs/editmode-test-results.xml"
+EDITMODE_TEST_RESULTS_FILE="${PROJECT_DIR}/logs/nunit/editmode-test-results.xml"
 
 echo "--- Testing Unity: Editmode :writing_hand:"