veristat-kernel: dump verifier log on failure

If veristat reports a verification failure of a program that's
expected to pass the verification according to the baseline, run
veristat on it again with -v to extract the verification log.

Signed-off-by: Ihor Solodrai <isolodrai@meta.com>

Author: theihor

.github/scripts/compare-veristat-results.sh

@@ -7,12 +7,36 @@ if [[ ! -f "${BASELINE_PATH}" ]]; then
     echo "Printing veristat results"
     cat "${VERISTAT_OUTPUT}"
 
-    exit
+    exit 0
 fi
 
-selftests/bpf/veristat \
+veristat=$(realpath selftests/bpf/veristat)
+cmp_out=$(mktemp veristate_compare_out_XXXXXX.csv)
+
+$veristat \
     --output-format csv \
     --emit file,prog,verdict,states \
-    --compare "${BASELINE_PATH}" "${VERISTAT_OUTPUT}" > compare.csv
+    --compare "${BASELINE_PATH}" "${VERISTAT_OUTPUT}" > $cmp_out
+
+python3 ./.github/scripts/veristat_compare.py $cmp_out
+exit_code=$?
+
+echo
+# if comparison failed, print verifier log for failure mismatches
+if [[ -n "$VERISTAT_DUMP_LOG_ON_FAILURE" && $exit_code -ne 0 ]]; then
+    cat $cmp_out | tail -n +1 | \
+    while read -r line; do
+        verdict=$(echo $line | cut -d',' -f4)
+        verdict_diff=$(echo $line | cut -d',' -f5)
+        if [[ "$verdict" == "failure" && "$verdict_diff" == "MISMATCH" ]]; then
+            file=$(echo $line | cut -d',' -f1)
+            prog=$(echo $line | cut -d',' -f2)
+            echo "VERIFIER LOG FOR $file/$prog:"
+            echo "=================================================================="
+            $veristat -v $VERISTAT_OBJECTS_DIR/$file -f $prog 2>&1
+            echo "=================================================================="
+        fi
+    done
+fi
 
-python3 ./.github/scripts/veristat_compare.py compare.csv
+exit $exit_code

.github/workflows/veristat-kernel.yml

@@ -30,6 +30,7 @@ jobs:
       REPO_PATH: ""
       KBUILD_OUTPUT: kbuild-output/
       ARCH_AND_TOOL: ${{ inputs.arch }}-${{ inputs.toolchain_full }}
+      VERISTAT_DUMP_LOG_ON_FAILURE: 'true'
 
     steps:
 
@@ -62,4 +63,3 @@ jobs:
         with:
           veristat_output: veristat-kernel
           baseline_name: ${{ env.ARCH_AND_TOOL}}-baseline-veristat-kernel
-