Merge pull request NixOS#12372 from roberth/test-characterisation-log-to-stderr

tests/functional/characterisation/framework: Log to stderr

Author: edolstra

tests/functional/characterisation/framework.sh

@@ -1,5 +1,7 @@
 # shellcheck shell=bash
 
+badTestNames=()
+
 # Golden test support
 #
 # Test that the output of the given test matches what is expected. If
@@ -18,10 +20,11 @@ function diffAndAcceptInner() {
     fi
 
     # Diff so we get a nice message
-    if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then
-        echo "FAIL: evaluation result of $testName not as expected"
+    if ! diff >&2 --color=always --unified "$expectedOrEmpty" "$got"; then
+        echo >&2 "FAIL: evaluation result of $testName not as expected"
         # shellcheck disable=SC2034
         badDiff=1
+        badTestNames+=("$testName")
     fi
 
     # Update expected if `_NIX_TEST_ACCEPT` is non-empty.
@@ -42,14 +45,14 @@ function characterisationTestExit() {
     if test -n "${_NIX_TEST_ACCEPT-}"; then
         if (( "$badDiff" )); then
             set +x
-            echo 'Output did mot match, but accepted output as the persisted expected output.'
-            echo 'That means the next time the tests are run, they should pass.'
+            echo >&2 'Output did mot match, but accepted output as the persisted expected output.'
+            echo >&2 'That means the next time the tests are run, they should pass.'
             set -x
         else
             set +x
-            echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,'
-            echo 'indicating the unexpected output should be accepted as the expected output going forward,'
-            echo 'but no tests had unexpected output so there was no expected output to update.'
+            echo >&2 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,'
+            echo >&2 'indicating the unexpected output should be accepted as the expected output going forward,'
+            echo >&2 'but no tests had unexpected output so there was no expected output to update.'
             set -x
         fi
         if (( "$badExitCode" )); then
@@ -60,16 +63,21 @@ function characterisationTestExit() {
     else
         if (( "$badDiff" )); then
             set +x
-            echo ''
-            echo 'You can rerun this test with:'
-            echo ''
-            echo "    _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.sh.test"
-            echo ''
-            echo 'to regenerate the files containing the expected output,'
-            echo 'and then view the git diff to decide whether a change is'
-            echo 'good/intentional or bad/unintentional.'
-            echo 'If the diff contains arbitrary or impure information,'
-            echo 'please improve the normalization that the test applies to the output.'
+            echo >&2 ''
+            echo >&2 'The following tests had unexpected output:'
+            for testName in "${badTestNames[@]}"; do
+                echo >&2 "    $testName"
+            done
+            echo >&2 ''
+            echo >&2 'You can rerun this test with:'
+            echo >&2 ''
+            echo >&2 "    _NIX_TEST_ACCEPT=1 meson test ${TEST_NAME}"
+            echo >&2 ''
+            echo >&2 'to regenerate the files containing the expected output,'
+            echo >&2 'and then view the git diff to decide whether a change is'
+            echo >&2 'good/intentional or bad/unintentional.'
+            echo >&2 'If the diff contains arbitrary or impure information,'
+            echo >&2 'please improve the normalization that the test applies to the output.'
             set -x
         fi
         exit $(( "$badExitCode" + "$badDiff" ))