Merge pull request #5091 from reinhapa/backport-build

[6.x.x] Backport build scripts

Author: adamretter

.github/actions/install-mvnd/action.yml

@@ -0,0 +1,89 @@
+name: 'install-mvnd'
+description: 'Install the Maven Daemon'
+inputs:
+  version:
+    description: 'The version of the Maven Daemon to install'
+    required: true
+    default: '0.9.0'
+  file-version-suffix:
+    description: 'A suffix to append to the version of the download file of Maven Daemon to install'
+    required: false
+    default: ''
+  install-path:
+    description: 'The folder in which Maven Daemon will be installed as a sub-folder'
+    required: true
+    default: '/tmp'
+  cache:
+    description: 'Set to true to cache Maven Daemon artifacts per-platform-architecture'
+    required: true
+    default: 'true'
+  mvnd-connect-timeout:
+    description: 'The timeout (as a duration, e.g. `90 seconds`) for connecting to the Maven Daemon'
+    required: true
+    default: '90 seconds'
+outputs:
+  mvnd-dir:
+    description: "The directory where the command mvnd is located"
+    value: ${{ steps.mvnd-location.outputs.mvnd-dir }}
+runs:
+  using: "composite"
+  steps:
+    - name: Determine mvnd platform and architecture
+      shell: bash
+      run: |
+        if [ "$RUNNER_OS" == "Linux" ]; then
+          MVND_PLATFORM="linux"
+        elif [ "$RUNNER_OS" == "macOS" ]; then
+          MVND_PLATFORM="darwin"
+        elif [ "$RUNNER_OS" == "Windows" ]; then
+          MVND_PLATFORM="windows"
+        else
+          "echo Unknown platform: $RUNNER_OS"
+          exit 1
+        fi
+        MVND_ARCHITECTURE="amd64"
+        echo "MVND_PLATFORM=${MVND_PLATFORM}" >> $GITHUB_ENV
+        echo "MVND_ARCHITECTURE=${MVND_ARCHITECTURE}" >> $GITHUB_ENV
+        echo "MVND_NAME=maven-mvnd-${{ inputs.version }}${{ inputs.file-version-suffix }}-${MVND_PLATFORM}-${MVND_ARCHITECTURE}" >> $GITHUB_ENV
+    - name: Cache mvnd
+      if: inputs.cache == 'true'
+      id: cache-mvnd
+      uses: actions/cache@v3
+      with:
+        path: |
+          ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip
+          ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256
+          ${{ inputs.install-path }}/${{ env.MVND_NAME }}
+        key: setup-${{ env.MVND_NAME }}
+    - name: Download mvnd
+      if: steps.cache-mvnd.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip
+        curl -fsSL -o ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256 https://archive.apache.org/dist/maven/mvnd/${{ inputs.version }}/${{ env.MVND_NAME }}.zip.sha256
+    - name: Install sha256sum (macOS)
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: brew install coreutils
+    - name: Verify mvnd sha256 checksum
+      shell: bash
+      run: echo "$(cat ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip.sha256) ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip" | sha256sum --check
+    - name: Unzip mvnd
+      if: steps.cache-mvnd.outputs.cache-hit != 'true'
+      shell: bash
+      run: unzip ${{ inputs.install-path }}/${{ env.MVND_NAME }}.zip -d ${{ inputs.install-path }}/
+    - name: Show Maven Daemon version
+      shell: bash
+      run: |
+        ${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin/mvnd -D'mvnd.connectTimeout=${{ inputs.mvnd-connect-timeout }}' --version
+        ${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin/mvnd -D'mvnd.connectTimeout=${{ inputs.mvnd-connect-timeout }}' --status
+    - name: Set mvnd-dir
+      id: mvnd-location
+      shell: bash
+      run: |
+        MVND_BIN_DIR="${{ inputs.install-path }}/${{ env.MVND_NAME }}/bin"
+        if [ "$RUNNER_OS" == "Windows" ]; then
+          MVND_BIN_DIR="$(cygpath --absolute --long-name --windows $MVND_BIN_DIR)"
+        fi
+        echo "MVND_BIN_DIR=${MVND_BIN_DIR}" >> $GITHUB_ENV
+        echo "mvnd-dir=${MVND_BIN_DIR}" >> $GITHUB_OUTPUT

.github/dependabot.yml

@@ -1,9 +1,16 @@
 version: 2
 updates:
-- package-ecosystem: maven
+
+- package-ecosystem: "github-actions"
+  target-branch: "develop-6.x.x"
   directory: "/"
   schedule:
-    interval: daily
+    interval: "weekly"
+- package-ecosystem: "maven"
+  target-branch: "develop-6.x.x"
+  directory: "/"
+  schedule:
+    interval: "daily"
     time: "03:00"
   open-pull-requests-limit: 10
   ignore:
@@ -36,23 +43,23 @@ updates:
     - "> 1.9.4, < 1.10"
   - dependency-name: net.sf.saxon:Saxon-HE
     versions:
-    > "9"
+    - "> 9"
   - dependency-name: org.hsqldb:hsqldb
     versions:
-    - 2.6.0
+    - "2.6.0"
   - dependency-name: com.ibm.icu:icu4j
     versions:
     - "69.1"
   - dependency-name: com.sun.mail:jakarta.mail
     versions:
-    - 2.0.1
+    - "2.0.1"
   - dependency-name: com.sun.activation:jakarta.activation
     versions:
-    - 2.0.1
+    - "2.0.1"
   - dependency-name: com.mycila:license-maven-plugin
     versions:
     - "4.0"
-    - 4.0.rc2
+    - "4.0.rc2"
   - dependency-name: org.owasp:dependency-check-maven
     versions:
-    - 6.1.3
+    - "6.1.3"

.github/workflows/ci-deploy.yml

@@ -7,29 +7,29 @@ jobs:
     # NOTE (DP): Publish on develop and master, test on PRs against these
     if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/master' || github.base_ref ==  'develop' || github.base_ref ==  'master'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
       - name: Set up JDK 8
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: temurin
           java-version: '8'
       - name: Make buildkit default
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
         id: buildx
         with:
           install: true
       - name: Cache Maven packages
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.m2
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
-          restore-keys: ${{ runner.os }}-m2
+          key: deploy-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: deploy-${{ runner.os }}-maven
       - name: Install bats
         run: sudo apt-get install bats    
       - name: Build images
-        run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true clean package
+        run: mvn -V -B -q -Pdocker -DskipTests -Ddependency-check.skip=true -P !mac-dmg-on-unix,!installer,!concurrency-stress-tests,!micro-benchmarks,skip-build-dist-archives clean package
       - name: Check local images
         run: docker image ls
       - name: Check license headers
@@ -66,6 +66,6 @@ jobs:
       #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       #     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       #     DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      #   run: mvn -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
+      #   run: mvn -q -Ddocker.tag=experimental -Ddocker.username=$DOCKER_USERNAME -Ddocker.password=$DOCKER_PASSWORD docker:build docker:push
       #   working-directory: ./exist-docker  
 

.github/workflows/ci-docs.yml

@@ -1,45 +1,80 @@
-name: Test
+name: Test & documentation
 on: [push, pull_request]
+permissions:
+  contents: read
+env:
+  MAVEN_OPTS: -XX:StartFlightRecording=maxsize=5g,disk=true,dumponexit=true,settings=default,filename=./ -DtrimStackTrace=false -D'maven.resolver.transport=wagon'
+  DEV_JDK: '8'
 jobs:
+  license:
+    name: License check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: ${{ env.DEV_JDK }}
+          cache: 'maven'
+      - run: mvn -V -B license:check
+        timeout-minutes: 60
+  dependencies:
+    name: Dependency checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v3
+        with:
+          distribution: temurin
+          java-version: ${{ env.DEV_JDK }}
+          cache: 'maven'
+      - run: mvn -V -B dependency-check:check
+        timeout-minutes: 60
   test:
-    name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Test
-    env:
-      MAVEN_OPTS: -XX:+IgnoreUnrecognizedVMOptions --illegal-access=debug -XX:StartFlightRecording=maxsize=5g,disk=true,dumponexit=true,settings=default,filename=./
+    name: ${{ matrix.os }} Test
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        jdk: ['8', '11']
-        include:
-          - os: ubuntu-latest
-            jdk: '16'
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up JDK ${{ matrix.jdk }}
-        uses: actions/setup-java@v2
+      - uses: actions/checkout@v4
+      - name: Set up JDK
+        uses: actions/setup-java@v3
         with:
           distribution: temurin
-          java-version: ${{ matrix.jdk }}
+          java-version: ${{ env.DEV_JDK }}
           cache: 'maven'
+      - name: Install Maven Daemon
+        id: install-mvnd
+        uses: ./.github/actions/install-mvnd
+        with:
+          version: '1.0-m7'
+          file-version-suffix: '-m39'
+          cache: 'true'
+      - name: Maven Build
+        timeout-minutes: 10
+        run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -T 1C compile test-compile -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip'
       - name: Maven Test
         timeout-minutes: 60
-        run: mvn -V -B -DtrimStackTrace=false clean verify
-        continue-on-error: ${{ matrix.jdk == '16' }}
-      - name: Maven Code Coverage
-        if: ${{ github.ref == 'refs/heads/develop' && matrix.jdk == '8' && matrix.os == 'ubuntu-latest' }}
+        run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B verify -DtrimStackTrace=false -D'dependency-check.skip' -D'license.skip' -D'mvnd.maxLostKeepAlive=6000'
+      - name: Javadoc (Linux only)
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B -q -T 1C install javadoc:javadoc -DskipTests -D'dependency-check.skip' -D'license.skip' --projects '!exist-distribution,!exist-installer' --also-make
+      - name: Maven Code Coverage (Develop branch on Linux only)
+        if: ${{ github.ref == 'refs/heads/develop' && matrix.os == 'ubuntu-latest' }}
         env:
           CI_NAME: github
           BRANCH_NAME_OR_REF: ${{ github.head_ref || github.ref }}
           CI_BUILD_NUMBER: ${{ github.run_id }}
           CI_BUILD_URL: https://github.com/${{ github.repository }}/commit/${{ github.event.after }}/checks
           COVERALLS_TOKEN: ${{ secrets.COVERALLS_TOKEN }}
-        run: mvn -V -B jacoco:report coveralls:report
+        run: ${{ steps.install-mvnd.outputs.mvnd-dir }}/mvnd -V -B jacoco:report coveralls:report
       - name: Archive build logs
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ runner.os }}-jdk${{ matrix.jdk }}-build-logs
+          name: ${{ runner.os }}-build-logs
           retention-days: 5
           path: |
             **/*.jfr

.github/workflows/ci-test.yml

@@ -8,24 +8,58 @@ jobs:
     name: W3C XQuery Test Suite
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v3
         with:
           distribution: temurin
           java-version: '8'
-          cache: 'maven'
+      - name: Cache Maven packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.m2
+          key: xqts-${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: xqts-${{ runner.os }}-maven
       - name: Maven XQTS Build
         run: mvn -V -B clean package -DskipTests -Ddependency-check.skip=true --projects exist-xqts --also-make
       - name: Run XQTS
         timeout-minutes: 60
         env:
-          JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication
-        run: find exist-xqts/target -name exist-xqts-runner.sh -exec {} --xqts-version HEAD --output-dir /tmp/xqts-output \;
+          JAVA_OPTS: -XX:+UseG1GC -XX:+UseStringDeduplication -Xmx6g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -XX:+ExitOnOutOfMemoryError
+        run: find exist-xqts/target -name exist-xqts-runner.sh -exec {} --xqts-version HEAD --output-dir /tmp/xqts-output --exclude-test-case RangeExpr-411d,RangeExpr-409d,RangeExpr-408d,RangeExpr-409c,RangeExpr-408c,GenCompEq-21 \;
+      - name: Check for HeapDump
+        id: check_heapdump
+        uses: andstor/file-existence-action@v2
+        with:
+          files: "/tmp/*.hprof"
+      - name: Compress HeapDump
+        if: steps.check_heapdump.outputs.files_exists == 'true'
+        run: zstd --rm -9 --progress -T0 /tmp/*.hprof
+      - name: Attach HeapDump artifact
+        if: steps.check_heapdump.outputs.files_exists == 'true'
+        uses: actions/upload-artifact@v3
+        with:
+          name: exist-xqts-runner-hprof
+          retention-days: 1
+          path: /tmp/*.hprof.zst
       - name: Archive XQTS Logs
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: xqts-logs
           retention-days: 14
           path: /tmp/xqts-output
+      - name: Get Previous XQTS Logs Artifacts JSON
+        run: 'curl -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/exist-db/exist/actions/artifacts?name=xqts-logs > /tmp/previous-xqts-logs-artifacts.json'
+      - name: Extract Previous XQTS Logs Artifact JSON
+        run: cat /tmp/previous-xqts-logs-artifacts.json | jq -r "[.artifacts[] | select(.workflow_run.head_branch == \"develop\")][1].archive_download_url" > /tmp/previous-xqts-logs-artifact.json
+      - name: Get Previous XQTS Logs Artifact
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: 'cat /tmp/previous-xqts-logs-artifact.json | xargs curl -H "Authorization: Bearer ${GITHUB_TOKEN}" --location --output /tmp/previous-xqts-output.zip'
+      - name: Extract Previous XQTS Logs Artifact
+        run: mkdir /tmp/previous-xqts-output && unzip /tmp/previous-xqts-output.zip -d /tmp/previous-xqts-output
+      - name: Compare Previous and Current XQTS Logs
+        run: java -jar ~/.m2/repository/net/sf/saxon/Saxon-HE/9.9.1-8/Saxon-HE-9.9.1-8.jar -xsl:exist-xqts/src/main/xslt/compare-results.xslt -it:compare-results -o:/tmp/comparison-results.xml xqts.previous.junit-data-path=/tmp/previous-xqts-output/junit/data xqts.current.junit-data-path=/tmp/xqts-output/junit/data
+      - name: Show Comparison Results
+        run: cat /tmp/comparison-results.xml