aswinshakil
diff --git a/‎.asf.yaml‎
Lines changed: 1 addition & 1 deletion b/‎.asf.yaml‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎.github/workflows/ci.yml‎
Lines changed: 135 additions & 33 deletions b/‎.github/workflows/ci.yml‎
Lines changed: 135 additions & 33 deletions
diff --git a/‎.github/workflows/intermittent-test-check.yml‎
Lines changed: 2 additions & 2 deletions b/‎.github/workflows/intermittent-test-check.yml‎
Lines changed: 2 additions & 2 deletions
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 github:
-  description: "Scalable, redundant, and distributed object store for Apache Hadoop"
+  description: "Scalable, reliable, distributed storage system optimized for data analytics and object store workloads."
   homepage: https://ozone.apache.org
   labels:
     - hadoop
 
@@ -28,8 +28,14 @@ on:
         required: false
 env:
   FAIL_FAST: ${{ github.event_name == 'pull_request' }}
+  # Minimum required Java version for running Ozone is defined in pom.xml (javac.version).
+  TEST_JAVA_VERSION: 21 # JDK version used by CI build and tests; should match the JDK version in apache/ozone-runner image
+  MAVEN_ARGS: --batch-mode --settings ${{ github.workspace }}/dev-support/ci/maven-settings.xml --show-version
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-  OZONE_WITH_COVERAGE: ${{ github.repository == 'apache/ozone' && github.event_name == 'push' }}
+  HADOOP_IMAGE: ghcr.io/apache/hadoop
+  OZONE_IMAGE: ghcr.io/apache/ozone
+  OZONE_RUNNER_IMAGE: ghcr.io/apache/ozone-runner
+  OZONE_WITH_COVERAGE: ${{ github.event_name == 'push' }}
 jobs:
   build-info:
     runs-on: ubuntu-20.04
@@ -102,10 +108,6 @@ jobs:
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: needs.build-info.outputs.needs-build == 'true'
-    strategy:
-      matrix:
-        java: [ 8 ]
-      fail-fast: false
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
@@ -136,11 +138,11 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: ${{ matrix.java }}
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Run a full build
         run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
         env:
@@ -155,6 +157,7 @@ jobs:
           retention-days: 1
       - name: Store source tarball for compilation
         uses: actions/upload-artifact@v4
+        if: needs.build-info.outputs.needs-compile == 'true'
         with:
           name: ozone-src
           path: hadoop-ozone/dist/target/ozone-*-src.tar.gz
@@ -171,6 +174,8 @@ jobs:
       - build-info
       - build
       - basic
+      - dependency
+      - license
     timeout-minutes: 45
     if: needs.build-info.outputs.needs-compile == 'true'
     strategy:
@@ -179,7 +184,7 @@ jobs:
         include:
           - os: ubuntu-20.04
           - java: 8
-            os: macos-12
+            os: macos-13
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
@@ -212,13 +217,13 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ matrix.java }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
       - name: Compile Ozone using Java ${{ matrix.java }}
-        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -Dskip.npx -Dskip.installnpx -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
+        run: hadoop-ozone/dev-support/checks/build.sh -Pdist -DskipRecon -Dmaven.javadoc.failOnWarnings=${{ matrix.java != 8 }} -Djavac.version=${{ matrix.java }} ${{ inputs.ratis_args }}
         env:
           OZONE_WITH_COVERAGE: false
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
@@ -253,27 +258,26 @@ jobs:
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             maven-repo-
-        if: ${{ !contains('author,bats,docs', matrix.check) }}
+        if: ${{ !contains('author,bats', matrix.check) }}
       - name: Download Ratis repo
         if: ${{ inputs.ratis_args != '' }}
         uses: actions/download-artifact@v4
         with:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java 8
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: 8
       - name: Execute tests
         run: hadoop-ozone/dev-support/checks/${{ matrix.check }}.sh ${{ inputs.ratis_args }}
-        continue-on-error: true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ matrix.check }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -309,19 +313,18 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: 8
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
         run: hadoop-ozone/dev-support/checks/${{ github.job }}.sh ${{ inputs.ratis_args }}
-        continue-on-error: true
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: ${{ !cancelled() }}
@@ -341,6 +344,15 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.sha }}
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -388,13 +400,76 @@ jobs:
           name: ozone-repo
           path: |
             ~/.m2/repository/org/apache/ozone
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
         run: |
           hadoop-ozone/dev-support/checks/${{ github.job }}.sh
+      - name: Summary of failures
+        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        if: ${{ failure() }}
+      - name: Archive build results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: ${{ github.job }}
+          path: target/${{ github.job }}
         continue-on-error: true
+  repro:
+    needs:
+      - build-info
+      - build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Download Ozone repo
+        id: download-ozone-repo
+        uses: actions/download-artifact@v4
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+      - name: Download Ratis repo
+        if: ${{ inputs.ratis_args != '' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: ratis-jars
+          path: |
+            ~/.m2/repository/org/apache/ratis
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: ${{ env.TEST_JAVA_VERSION }}
+      - name: Execute tests
+        run: |
+          hadoop-ozone/dev-support/checks/${{ github.job }}.sh -Pdist -Psrc -Dmaven.javadoc.skip=true ${{ inputs.ratis_args }}
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
+      - name: Install diffoscope
+        run: |
+          sudo apt update -q
+          sudo apt install -y diffoscope
+        if: ${{ failure() }}
+      - name: Check artifact differences
+        run: |
+          hadoop-ozone/dev-support/checks/_diffoscope.sh
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -407,6 +482,8 @@ jobs:
       - build-info
       - build
       - basic
+      - dependency
+      - license
     runs-on: ubuntu-20.04
     timeout-minutes: 150
     if: needs.build-info.outputs.needs-compose-tests == 'true'
@@ -419,6 +496,15 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.sha }}
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -438,10 +524,9 @@ jobs:
           KEEP_IMAGE: false
           OZONE_ACCEPTANCE_SUITE: ${{ matrix.suite }}
           OZONE_VOLUME_OWNER: 1000
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -454,6 +539,8 @@ jobs:
       - build-info
       - build
       - basic
+      - dependency
+      - license
     runs-on: ubuntu-20.04
     timeout-minutes: 60
     if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
@@ -462,6 +549,15 @@ jobs:
         uses: actions/checkout@v4
         with:
           ref: ${{ needs.build-info.outputs.sha }}
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -476,10 +572,9 @@ jobs:
           sudo mkdir .aws && sudo chmod 777 .aws && sudo chown 1000 .aws
           popd
           ./hadoop-ozone/dev-support/checks/kubernetes.sh
-        continue-on-error: true
       - name: Summary of failures
         run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -529,25 +624,31 @@ jobs:
           name: ratis-jars
           path: |
             ~/.m2/repository/org/apache/ratis
-      - name: Setup java
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.TEST_JAVA_VERSION }}
       - name: Execute tests
-        continue-on-error: true
         run: |
           args="${{ inputs.ratis_args }}"
           if [[ "${{ matrix.profile }}" == "flaky" ]]; then
             args="$args -Dsurefire.rerunFailingTestsCount=5 -Dsurefire.fork.timeout=3600"
           fi
+          if [[ "${{ matrix.profile }}" != "filesystem" ]]; then
+            args="$args -DskipShade"
+          fi
 
           hadoop-ozone/dev-support/checks/integration.sh -P${{ matrix.profile }} ${args}
         env:
           DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
       - name: Summary of failures
-        run: hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
-        if: ${{ !cancelled() }}
+        run: |
+          if [[ -s "target/${{ github.job }}/summary.md" ]]; then
+            cat target/${{ github.job }}/summary.md >> $GITHUB_STEP_SUMMARY
+          fi
+          hadoop-ozone/dev-support/checks/_summary.sh target/${{ github.job }}/summary.txt
+        if: ${{ failure() }}
       - name: Archive build results
         uses: actions/upload-artifact@v4
         if: always()
@@ -558,7 +659,7 @@ jobs:
   coverage:
     runs-on: ubuntu-20.04
     timeout-minutes: 30
-    if: github.repository == 'apache/ozone' && github.event_name == 'push'
+    if: github.event_name == 'push'
     needs:
       - build-info
       - acceptance
@@ -587,15 +688,16 @@ jobs:
         run: |
           mkdir -p hadoop-ozone/dist/target
           tar xzvf target/artifacts/ozone-bin/ozone*.tar.gz -C hadoop-ozone/dist/target
-      - name: Calculate combined coverage
-        run: ./hadoop-ozone/dev-support/checks/coverage.sh
-      - name: Setup java 17
+      - name: Setup java ${{ env.TEST_JAVA_VERSION }}
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.TEST_JAVA_VERSION }}
+      - name: Calculate combined coverage
+        run: ./hadoop-ozone/dev-support/checks/coverage.sh
       - name: Upload coverage to Sonar
         run: ./hadoop-ozone/dev-support/checks/sonar.sh
+        if: github.repository == 'apache/ozone'
         env:
           SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -115,7 +115,7 @@ jobs:
           java-version: 8
       - name: Build (most) of Ozone
         run: |
-          args="-Dskip.npx -Dskip.installnpx -DskipShade -Dmaven.javadoc.skip=true"
+          args="-DskipRecon -DskipShade -Dmaven.javadoc.skip=true"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"
@@ -183,7 +183,7 @@ jobs:
             export OZONE_REPO_CACHED=true
           fi
 
-          args="-DexcludedGroups=native|slow|unhealthy"
+          args="-DexcludedGroups=native|slow|unhealthy -DskipShade"
           if [[ "${{ github.event.inputs.ratis-ref }}" != "" ]]; then
             args="$args -Dratis.version=${{ needs.ratis.outputs.ratis-version }}"
             args="$args -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }}"