build: don't exit after first ingest test failure (#166)

so more diff's may be inspected.

Author: cragwolfe

.github/workflows/ci.yml

@@ -143,7 +143,8 @@ jobs:
         make install-ingest-gitlab
         make install-ingest-slack
         make install-ingest-wikipedia
-        ./test_unstructured_ingest/test-ingest.sh
+        # only run ingest tests that check expected output diffs.
+        bash inference/scripts/test-unstructured-ingest-helper.sh
 
   changelog:
     runs-on: ubuntu-latest

scripts/test-unstructured-ingest-helper.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# This is intended to be run from an unstructured checkout, not in this repo
+# The goal here is to see what changes the current branch would introduce to unstructured
+# fixtures
+
+INGEST_COMMANDS=(
+    test_unstructured_ingest/test-ingest-azure.sh
+    test_unstructured_ingest/test-ingest-biomed-api.sh
+    test_unstructured_ingest/test-ingest-biomed-path.sh
+    test_unstructured_ingest/test-ingest-box.sh
+    test_unstructured_ingest/test-ingest-dropbox.sh
+    test_unstructured_ingest/test-ingest-gcs.sh
+    test_unstructured_ingest/test-ingest-onedrive.sh
+    test_unstructured_ingest/test-ingest-s3.sh
+)
+
+EXIT_STATUSES=()
+
+# Run each command and capture its exit status
+for INGEST_COMMAND in "${INGEST_COMMANDS[@]}"; do
+  $INGEST_COMMAND
+  EXIT_STATUSES+=($?)
+done
+
+# Check for failures
+for STATUS in "${EXIT_STATUSES[@]}"; do
+  if [[ $STATUS -ne 0 ]]; then
+    echo "At least one ingest command failed! Scroll up to see which"
+    exit 1
+  fi
+done
+
+echo "No diff's resulted from any ingest commands"