[ci] Increase reliability by capturing the failure configuration (#96)

This chance improves the testing failures due tests that we expect will fail at that stage.

Author: aaronj0

.github/workflows/ci.yml

@@ -336,8 +336,8 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         # Install deps
-        sudo apt update
-        sudo apt install git g++ debhelper devscripts gnupg python3 valgrind
+        sudo apt-get update
+        sudo apt-get install git g++ debhelper devscripts gnupg python3 valgrind
         conda install -y -q -c conda-forge \
           distro \
           pytest
@@ -716,7 +716,6 @@ jobs:
         # We need PYTHONPATH later
         echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
     - name: Run the tests on Unix Systems
-      continue-on-error: true
       if: ${{ (runner.os != 'windows') && (matrix.cppyy == 'On') }}
       run: |
         cd ..
@@ -728,20 +727,23 @@ jobs:
         python -m pip install --upgrade pip
         python -m pip install pytest
         python -m pip install pytest-xdist
+        python -m pip install numba
         echo ::endgroup::
+        echo ::group::Run complete test suite
+        python -m pytest -sv | tee complete_testrun.log 2>&1
         echo ::group::Crashing Test Logs
         # See if we don't have a crash that went away
         # Comment out all xfails but the ones that have a run=False condition.
         find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \;
-        python -m pytest -n 1 -m "xfail" --runxfail -sv --max-worker-restart 512 | tee test_crashed.log 2>&1
+        python -m pytest -n 1 -m "xfail" --runxfail -sv --max-worker-restart 512 | tee test_crashed.log 2>&1 || true
         git checkout .
         echo ::endgroup::
         echo ::group::XFAIL Test Logs
         # Rewrite all xfails that have a run clause to skipif. This way we will
         # avoid conditionally crashing xfails
         find . -name "*.py" -exec sed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\[email protected](condition=not \2/g' {} \;
         # See if we don't have an xfail that went away
-        python -m pytest --runxfail -sv  | tee test_xfailed.log 2>&1
+        python -m pytest --runxfail -sv  | tee test_xfailed.log 2>&1 || true
         git checkout .
         echo ::endgroup::
         echo ::group::Passing Test Logs
@@ -750,12 +752,20 @@ jobs:
         declare -i RETCODE=0
 
         set -o pipefail
-        valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v
+        if [[ "${{ matrix.clang-runtime }}" == "17" || "${{ matrix.clang-runtime }}" == "18" ]] && [[ "${{ matrix.os }}" != "macos-14" ]]; then
+          echo "Valgrind reports true for clang-runtime 17 or 18, due to memory leaks with LLVM"
+          valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v || true
+        else
+          echo "Running valgrind on passing tests"
+          valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v
+        fi
         export RETCODE=+$?
         echo ::endgroup::
 
         RETCODE=+$?
 
+        echo "Complete Test Suite Summary: \n"
+        tail -n1 complete_testrun.log
         echo "Crashing Summary: \n"
         tail -n1 test_crashed.log
         echo "XFAIL Summary:"