chore: fix parallelism in library compilation (#26395)

* chore: fix parallelism in library compilation

* trigger on pull request (temporarily)

* do not use awk

* try running command without piping output

* move to script file

* chore: add vim files to .gitignore

* chore: skip enforcer in verification script

* specify working directory

* parallelize on variant

* fix "run" step

* do not cancel parallel jobs if one fails

* parallelize by alphabet letter

* use explicit letters

* correct matrix config

* add runs-on to print_results job

* test against current variant only

* fix output file reference

* fix error handling

* add output consumption config

* use official recommendation for parallel job output handling

* correct output reference

* print results from parallel jobs

* fix sed command

* fail on error

* do not use xtrace on print results job

* improve error message

Author: diegomarquezp

.github/workflows/verify.sh

@@ -0,0 +1,40 @@
+set -ex
+export repo_dir=$(realpath $(dirname "${BASH_SOURCE[0]}")/../../)
+pushd "${repo_dir}/clients"
+
+# We only test against the latest variant
+readonly CURRENT_VARIANT="2.0.0"
+
+# An optional argument to only compile libraries starting with this letter
+starting_letter="$1"
+
+# find all generated clients' pom.xml
+find . -wholename "*${CURRENT_VARIANT}/pom.xml" -not -path '*/target/*' > pom_list_raw
+cat pom_list_raw
+
+# trim down to those starting with "${starting_letter}"
+cat pom_list_raw | { grep "google-api-services-${starting_letter}"  || true; } > pom_list
+cat pom_list
+
+
+# format result to list of Maven modules
+cat pom_list | cut -d'/' -f2-4 \
+| sed 's/\(.*\)/<module>\1<\/module>/' > module_list
+cat module_list
+
+# produce a temporary pom with the modules
+(echo "<project><modelVersion>4.0.0</modelVersion><groupId>temp</groupId><artifactId>temp</artifactId><version>1.0</version><packaging>pom</packaging><modules>"; cat module_list; echo "</modules></project>") > pom.xml
+cat pom.xml
+
+# use generated pom to test compilation
+mvn clean compile -T 1.5C -Dmaven.testSkip=true -Denforcer.skip -fae --fail-at-end 2>&1 | tee out
+cat out | grep 'rev20' | grep 's]' | { grep 'FAILURE' || true; } > errors
+if [[ $(cat errors | wc -l) -gt 0 ]]; then
+	echo "Compilation errors found in the following libraries:"
+	cat errors
+	# send to GH output
+	if [[ -f "${GITHUB_OUTPUT}" ]]; then
+	  echo "failed_libraries_${starting_letter}=$(cat errors | tr '\n' ',')" > "${GITHUB_OUTPUT}"
+	fi
+fi
+echo "No compilation errors found"

.github/workflows/verify.yaml

@@ -3,12 +3,48 @@ on:
     # Runs at 04:00 am
     - cron: '0 4 * * *'
   workflow_dispatch:
+  # delete this before merging the PR
+  pull_request:
 # Generates a list of libraries that cannot be
 # compiled (printed to the action stdout)
 name: Verify libraries compilation
 jobs:
   verify:
     runs-on: 'ubuntu-24.04'
+    continue-on-error: true
+    strategy:
+      matrix:
+        # We will parallelize by alphabet letter
+        letter: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']
+      fail-fast: false
+    outputs:
+      # follows official example from https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/passing-information-between-jobs#using-job-outputs-in-a-matrix-job
+      failed_libraries_a: ${{ steps.compile.outputs.failed_libraries_a }}
+      failed_libraries_b: ${{ steps.compile.outputs.failed_libraries_b }}
+      failed_libraries_c: ${{ steps.compile.outputs.failed_libraries_c }}
+      failed_libraries_d: ${{ steps.compile.outputs.failed_libraries_d }}
+      failed_libraries_e: ${{ steps.compile.outputs.failed_libraries_e }}
+      failed_libraries_f: ${{ steps.compile.outputs.failed_libraries_f }}
+      failed_libraries_g: ${{ steps.compile.outputs.failed_libraries_g }}
+      failed_libraries_h: ${{ steps.compile.outputs.failed_libraries_h }}
+      failed_libraries_i: ${{ steps.compile.outputs.failed_libraries_i }}
+      failed_libraries_j: ${{ steps.compile.outputs.failed_libraries_j }}
+      failed_libraries_k: ${{ steps.compile.outputs.failed_libraries_k }}
+      failed_libraries_l: ${{ steps.compile.outputs.failed_libraries_l }}
+      failed_libraries_m: ${{ steps.compile.outputs.failed_libraries_m }}
+      failed_libraries_n: ${{ steps.compile.outputs.failed_libraries_n }}
+      failed_libraries_o: ${{ steps.compile.outputs.failed_libraries_o }}
+      failed_libraries_p: ${{ steps.compile.outputs.failed_libraries_p }}
+      failed_libraries_q: ${{ steps.compile.outputs.failed_libraries_q }}
+      failed_libraries_r: ${{ steps.compile.outputs.failed_libraries_r }}
+      failed_libraries_s: ${{ steps.compile.outputs.failed_libraries_s }}
+      failed_libraries_t: ${{ steps.compile.outputs.failed_libraries_t }}
+      failed_libraries_u: ${{ steps.compile.outputs.failed_libraries_u }}
+      failed_libraries_v: ${{ steps.compile.outputs.failed_libraries_v }}
+      failed_libraries_w: ${{ steps.compile.outputs.failed_libraries_w }}
+      failed_libraries_x: ${{ steps.compile.outputs.failed_libraries_x }}
+      failed_libraries_y: ${{ steps.compile.outputs.failed_libraries_y }}
+      failed_libraries_z: ${{ steps.compile.outputs.failed_libraries_z }}
     steps:
       - uses: actions/setup-java@v3
         with:
@@ -17,8 +53,27 @@ jobs:
       - uses: actions/checkout@v2
         with:
           path: google-api-java-client-services
-      # we install the moreutils `parallel` command
-      - run: sudo apt-get install moreutils
-      - working-directory: google-api-java-client-services
-        run: bash .github/workflows/verify_compilation.sh
+
+      - id: compile
+        working-directory: google-api-java-client-services
+        run: |
+          set -ex
+          bash .github/workflows/verify.sh "${{matrix.letter}}"
+  print_results:
+    runs-on: 'ubuntu-24.04'
+    needs: [verify]
+    steps:
+      - run: |
+          set -e
+          echo '${{ toJSON(needs.verify.outputs) }}' \
+          | jq -j 'to_entries[] | select(.key | startswith("failed_libraries_")) | .value' \
+          | sed 's/,/\n/g' > failed_libs
+
+          if [[ $(cat failed_libs | wc -l) -gt 0 ]]; then
+            echo "The following libraries cannot be compiled:"
+            cat failed_libs
+            exit 1
+          fi
+          echo "All libraries of the current variant are compilable!"
+
 

.github/workflows/verify_compilation.sh

@@ -1,5 +1,8 @@
 .gitignore
 
+# Vim files
+*.swp
+
 # Packages
 dist
 build