Speed up builds (#267)

- Speed up builds by uploading one bulk report tarball rather than
individual files.
- Switch out GZIP with XZ and use the maximum compression factor.

Author: ascopes

.gitattributes

@@ -0,0 +1,4 @@
+# Needed to stop Git Bash in CI accidentally injecting \r into jvm.config and breaking builds.
+*         text=auto     eol=lf
+*.cmd     text=auto     eol=crlf
+*.jar     -text

.github/workflows/build.yml

@@ -79,19 +79,7 @@ jobs:
 
       - name: Compile and run tests
         shell: bash
-        run: >-
-          ./mvnw 
-          -B 
-          -e
-          -T1C
-          -U
-          --no-transfer-progress
-          -Dcheckstyle.skip=true
-          -Dlicense.skip=true
-          -Dstyle.color=always
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-          ${{ inputs.maven-opts }}
-          clean verify
+        run: ./mvnw -B -U -Dcheckstyle.skip=true -Dlicense.skip=true clean verify
 
       - name: Annotate test reports with build environment info
         if: always()
@@ -113,8 +101,9 @@ jobs:
           set +f -x
           # Allow globs to not match anything without causing errors.
           shopt -s nullglob
-          
-          tar -czvf reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tgz \
+  
+          # XZ with max compression, more efficient than using GZip.  
+          XZ_OPT=-9 tar -Jcvf reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz \
               **/target/failsafe-reports/** \
               **/target/surefire-reports/** \
               **/target/site/jacoco/unit/jacoco*.xml \
@@ -124,10 +113,10 @@ jobs:
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: zzz-reports-${{ matrix.java-version }}-${{ matrix.os-name }}
+          name: reports-${{ matrix.java-version }}-${{ matrix.os-name }}
           if-no-files-found: error
-          path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tgz
-          retention-days: 1
+          path: reports-${{ matrix.java-version }}-${{ matrix.os-name }}.tar.xz
+          retention-days: 30
 
   mutation-tests:
     name: Run mutation tests
@@ -149,21 +138,7 @@ jobs:
       - name: Run mutation tests
         shell: bash
         # Do not run on multiple threads, the plugin does not support this properly.
-        run: >-
-          ./mvnw 
-          -B
-          -e
-          -T1
-          -U
-          --also-make
-          -pl java-compiler-testing
-          --no-transfer-progress
-          -P mutation
-          -Dstyle.color=always
-          -Dcheckstyle.skip=true
-          -Dlicense.skip=true
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-          test
+        run: ./mvnw -B -U -am -pl java-compiler-testing -P mutation -Dcheckstyle.skip=true -Dlicense.skip=true test
 
       # Compress first so that the collection job later takes far less time (order of a few minutes
       # or so). GitHub does not compress these until after the workflow finishes, meaning when
@@ -177,16 +152,16 @@ jobs:
           set +f -x
           # Allow globs to not match anything without causing errors.
           shopt -s nullglob
-          
-          tar -czvf reports-mutation-tests.tgz **/target/pit-reports/pit-reports/**
+          # XZ with max compression, more efficient than using GZip.
+          XZ_OPT=-9 tar -Jcvf reports-mutation-tests.tar.xz **/target/pit-reports/pit-reports/**
 
       - name: Stash reports tarball
         uses: actions/upload-artifact@v3
         if: always()
         with:
-          name: zzz-reports-mutation
+          name: reports-mutation-tests
           if-no-files-found: error
-          path: reports-mutation-tests.tgz
+          path: reports-mutation-tests.tar.xz
           retention-days: 30
 
   publish-test-reports:
@@ -211,19 +186,13 @@ jobs:
       - name: Download stashed tarballs
         uses: actions/download-artifact@v3
         with:
-          path: 'artifacts/reports-*.tgz'
+          path: 'artifacts/reports-*.tar.xz'
 
       - name: Decompress stashed report tarballs
         shell: bash
         run: |-
-          find artifacts/ -name "reports-*.tgz" -exec tar -xzvf {} \;
-          find artifacts/ -name "reports-*.tgz" -exec rm -v {} \;
-
-      - name: Delete temporary artifacts
-        uses: geekyeggo/delete-artifact@v2
-        with:
-          name: 'zzz-reports-*'
-          failOnError: false
+          find artifacts/ -name "reports-*.tar.xz" -exec tar -Jxvf {} \;
+          find artifacts/ -name "reports-*.tar.xz" -exec rm -v {} \;
 
       - name: Publish test results
         continue-on-error: true
@@ -261,25 +230,10 @@ jobs:
           chmod -v +x codecov
           ./codecov -v
           # TODO: upload unit test and integration test reports separately so that
-          # codecov processes it correctly. Need to hardwire the paths in here somehow.
+          # codecov processes it correctly. Need to hard-wire the paths in here somehow.
           #./codecov -c -F unit -v
           #./codecov -c -F integration -v
 
-      # Now we are finishing, GitHub will compress this properly.
-      - name: Upload all reports in one archive
-        uses: actions/upload-artifact@v3
-        if: always()
-        with:
-          name: test-reports
-          if-no-files-found: error
-          path: |-
-            **/target/pit-reports/pit-reports/**
-            **/target/failsafe-reports/**
-            **/target/surefire-reports/**
-            **/target/site/jacoco/unit/jacoco*.xml
-            **/target/site/jacoco/int/jacoco*.xml
-          retention-days: 30
-
   formatting:
     name: Check formatting and licenses
     runs-on: ubuntu-22.04
@@ -300,19 +254,12 @@ jobs:
       - name: Run checks
         shell: bash
         run: >-
-          ./mvnw 
-          -B
-          -e
-          -T1C
-          -U
-          --no-transfer-progress
+          ./mvnw -B -U
           -DskipTests=true
-          -Dstyle.color=always
           -Dmaven.main.skip
           -Dmaven.jar.skip
           -Dmaven.resources.skip
           -Dmaven.test.skip
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
           verify
 
   dependency-check:
@@ -336,21 +283,15 @@ jobs:
         shell: bash
         run: >-
           ./mvnw 
-          -B
-          -e
-          -T1C
-          -U
-          -Pdependency-check
-          --no-transfer-progress
+          -B -U
+          -P dependency-check
           -DskipTests=true
-          -Dstyle.color=always
           -Dcheckstyle.skip=true
           -Dlicense.skip=true
           -Dmaven.main.skip
           -Dmaven.jar.skip
           -Dmaven.resources.skip
           -Dmaven.test.skip
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
           verify
 
       - name: Archive Dependency Scan reports
@@ -382,19 +323,10 @@ jobs:
       - name: Generate JavaDocs
         shell: bash
         run: >-
-          ./mvnw 
-          -B
-          -e
-          -T1C
-          -U
-          -pl java-compiler-testing
-          --also-make
-          --no-transfer-progress
+          ./mvnw  -B -U -am -pl java-compiler-testing
           -Dmaven.test.skip=true
           -Dcheckstyle.skip=true
           -Dlicense.skip=true
-          -Dstyle.color=always
-          -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
           clean compile javadoc:jar
 
       - name: Upload JavaDocs as a build artifact
@@ -424,4 +356,3 @@ jobs:
         with:
           name: github-pages
           failOnError: false
-

.github/workflows/deploy.yml

@@ -62,7 +62,7 @@ jobs:
           
           info "Determining release version to use (this may take a moment)..."
           if [[ '${{ inputs.version }}' == "" ]]; then
-            release_version="$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT//g')"
+            release_version="$(./mvnw -B help:evaluate -Dexpression=project.version -q -DforceStdout | sed 's/-SNAPSHOT//g')"
           else
             release_version='${{ inputs.version }}'
           fi
@@ -83,14 +83,11 @@ jobs:
           run <<-SCRIPT          
             ./mvnw -B -e \
                 -Preleases \
-                --no-transfer-progress \
                 -Darguments='${build_args[@]}' \
                 -DdryRun='${{ inputs.dry-run }}' \
-                -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 \
                 -Dpassword='${{ secrets.GITHUB_TOKEN }}' \
                 -DreleaseVersion='${release_version}' \
                 -DsignTag=false \
-                -Dstyle.color=always \
                 -Dtag='v${release_version}' \
                 release:prepare release:perform
           SCRIPT

.mvn/jvm.config

@@ -0,0 +1 @@
+-XX:+TieredCompilation -XX:TieredStopAtLevel=1

.mvn/maven.config

@@ -0,0 +1 @@
+-e -T1C -Dstyle.color=always -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 --no-transfer-progress

mvnw

@@ -307,6 +307,8 @@ export MAVEN_CMD_LINE_ARGS
 
 WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain
 
+set -x
+
 exec "$JAVACMD" \
   $MAVEN_OPTS \
   $MAVEN_DEBUG_OPTS \

scripts/prepare-test-outputs-for-merge.sh

@@ -89,12 +89,12 @@ info "Generated XSLT script at ${surefire_prefix_xslt}"
 
 function find-all-surefire-reports {
   info "Discovering Surefire test reports"
-  find . -wholename '**/target/surefire-reports/TEST-*Test.xml' -print0 | xargs -0
+  find . -wholename '**/target/surefire-reports/TEST-*Test.xml' -print
 }
 
 function find-all-failsafe-reports {
   info "Discovering Failsafe test reports"
-  find . -wholename '**/target/failsafe-reports/TEST-*Test.xml' -print0 | xargs -0
+  find . -wholename '**/target/failsafe-reports/TEST-*Test.xml' -print
 }
 
 function find-all-jacoco-reports {
@@ -109,19 +109,39 @@ function find-all-jacoco-reports {
   fi
 }
 
+function xsltproc-surefire-report {
+  local prefix="${1}"
+  local xslt="${2}"
+  local input_report="${3}"
+  local output_report="${4}"
+
+  if ! run <<< "xsltproc --stringparam prefix '${prefix}' '${xslt}' '${input_report}' > '${output_report}'"; then
+    err "Error invoking xsltproc! Erroneous report was:"
+    dump "${input_report}"
+    return 2
+  fi
+
+  rm "${input_report}"
+}
+
 info "Updating test reports..."
 report_count=0
-for report in $(find-all-surefire-reports) $(find-all-failsafe-reports); do
+prefix="[Java-${ci_java_version}-${ci_os}]"
+concurrency="$(($(nproc) * 4))"
+
+while read -r report; do
   report_count="$((report_count+1))"
   new_report="${report/.xml/-java-${ci_java_version}-${ci_os}.xml}"
-  prefix="[Java-${ci_java_version}-${ci_os}]"
-  if ! run <<< "xsltproc --stringparam prefix '${prefix}' '${surefire_prefix_xslt}' '${report}' > '${new_report}'"; then
-    err "Error invoking xsltproc! Erroneous report was:"
-    dump "${report}"
-    exit 2
+  xsltproc-surefire-report "${prefix}" "${surefire_prefix_xslt}" "${report}" "${new_report}" &
+
+  # Wait if we have the max number of jobs in the window running (cpu-count * 4)
+  if [ "$((report_count % concurrency))" -eq 0 ]; then
+    wait < <(jobs -p)
+    info "Waited for up to ${concurrency} jobs to complete, will now continue..."
   fi
-  run <<< "rm '${report}'"
-done
+done < <(find-all-surefire-reports; find-all-failsafe-reports)
+wait < <(jobs -p)
+
 success "Updated ${report_count} test reports"
 
 info "Updating coverage reports..."