Merge remote-tracking branch 'upstream/main' into temp_decryption_buffers

Author: adamreeve

.env

@@ -47,9 +47,9 @@ ULIMIT_CORE=-1
 
 # Default versions for platforms
 ALMALINUX=8
-ALPINE_LINUX=3.18
+ALPINE_LINUX=3.22
 DEBIAN=12
-FEDORA=39
+FEDORA=42
 UBUNTU=22.04
 
 # Default versions for various dependencies
@@ -67,6 +67,7 @@ LLVM=18
 MAVEN=3.8.7
 NODE=20
 NUMBA=latest
+NUMBA_CUDA=latest
 NUMPY=latest
 PANDAS=latest
 PYTHON=3.9

.github/workflows/archery.yml

@@ -58,7 +58,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Git Fixup

.github/workflows/comment_bot.yml

@@ -36,7 +36,7 @@ jobs:
       pull-requests: write
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           path: arrow
           # fetch the tags for version number generation
@@ -49,7 +49,7 @@ jobs:
         run: pip install -e arrow/dev/archery[bot]
       - name: Handle GitHub comment event
         env:
-          ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }}
         run: |
           archery --debug trigger-bot \

.github/workflows/cpp.yml

@@ -110,7 +110,7 @@ jobs:
       UBUNTU: ${{ matrix.ubuntu }}
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
           submodules: recursive
@@ -163,7 +163,7 @@ jobs:
     timeout-minutes: 45
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive
@@ -218,7 +218,7 @@ jobs:
           sysctl -a | grep cpu
           sysctl -a | grep "hw.optional"
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive
@@ -334,7 +334,7 @@ jobs:
             /d 1 `
             /f
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive
@@ -431,7 +431,7 @@ jobs:
             /d 1 `
             /f
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
           submodules: recursive

.github/workflows/cpp_extra.yml

@@ -81,6 +81,9 @@ jobs:
     outputs:
       ci-extra: ${{ steps.check.outputs.ci-extra }}
     steps:
+      - name: Checkout Arrow
+        if: github.event_name == 'pull_request'
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - name: Check
         id: check
         env:
@@ -99,7 +102,15 @@ jobs:
               if [ "${n_ci_extra_labels}" -eq 1 ]; then
                 ci_extra=true
               else
-                ci_extra=false
+                git fetch origin ${GITHUB_BASE_REF}
+                if git diff --stat origin/${GITHUB_BASE_REF}.. | \
+                     grep \
+                       --fixed-strings ".github/workflows/cpp_extra.yml" \
+                       --quiet; then
+                  ci_extra=true
+                else
+                  ci_extra=false
+                fi
               fi
               ;;
           esac
@@ -124,13 +135,22 @@ jobs:
               -e ARROW_USE_MESON=ON
             runs-on: ubuntu-latest
             title: AMD64 Ubuntu Meson
+          # TODO: We should remove this "continue-on-error: true" once GH-47207 is resolved
+          - continue-on-error: true
+            envs:
+              - DEBIAN=13
+            image: debian-cpp
+            run-options: >-
+              -e CMAKE_CXX_STANDARD=23
+            runs-on: ubuntu-latest
+            title: AMD64 Debian C++23
     env:
       ARCHERY_DEBUG: 1
       ARROW_ENABLE_TIMING_TESTS: OFF
       DOCKER_VOLUME_PREFIX: ".docker/"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
           submodules: recursive
@@ -147,13 +167,20 @@ jobs:
       - name: Setup Archery
         run: python3 -m pip install -e dev/archery[docker]
       - name: Execute Docker Build
+        continue-on-error: ${{ matrix.continue-on-error || false }}
         env:
           ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
+          ENVS: ${{ toJSON(matrix.envs) }}
         run: |
           # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes
           sudo sysctl -w vm.mmap_rnd_bits=28
           source ci/scripts/util_enable_core_dumps.sh
+          if [ "${ENVS}" != "null" ]; then
+            echo "${ENVS}" | jq -r '.[]' | while read env; do
+              echo "${env}" >> .env
+            done
+          fi
           archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }}
       - name: Docker Push
         if: >-
@@ -166,3 +193,88 @@ jobs:
           ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
         continue-on-error: true
         run: archery docker push ${{ matrix.image }}
+
+  jni-macos:
+    needs: check-labels
+    name: JNI macOS
+    runs-on: macos-14
+    if: needs.check-labels.outputs.ci-extra == 'true'
+    timeout-minutes: 45
+    env:
+      MACOSX_DEPLOYMENT_TARGET: "14.0"
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          brew bundle --file=cpp/Brewfile
+          # We want to link aws-sdk-cpp statically but Homebrew's
+          # aws-sdk-cpp provides only shared library. If we have
+          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
+          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
+          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
+          brew uninstall aws-sdk-cpp
+          # We want to use bundled RE2 for static linking. If
+          # Homebrew's RE2 is installed, its header file may be used.
+          # We uninstall Homebrew's RE2 to ensure using bundled RE2.
+          brew uninstall grpc || : # gRPC depends on RE2
+          brew uninstall [email protected] || : # gRPC 1.54 may be installed too
+          brew uninstall re2
+          # We want to use bundled Protobuf for static linking. If
+          # Homebrew's Protobuf is installed, its library file may be
+          # used on test  We uninstall Homebrew's Protobuf to ensure using
+          # bundled Protobuf.
+          brew uninstall protobuf
+      - name: Prepare ccache
+        run: |
+          echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
+      - name: Cache ccache
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ccache
+          key: jni-macos-${{ hashFiles('cpp/**') }}
+          restore-keys: jni-macos-
+      - name: CMake
+        run: |
+          cmake \
+            -S cpp \
+            -B cpp.build \
+            --preset=ninja-release-jni-macos \
+            -DARROW_BUILD_TESTS=ON \
+            -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
+      - name: Build
+        run: |
+          cmake --build cpp.build
+      - name: Install
+        run: |
+          cmake --install cpp.build
+      - name: Test
+        env:
+          ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
+          PARQUET_TEST_DATA: ${{ github.workspace }}/cpp/submodules/parquet-testing/data
+        run: |
+          # MinIO is required
+          exclude_tests="arrow-s3fs-test"
+          # unstable
+          exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
+          exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
+          ctest \
+            --exclude-regex "${exclude_tests}" \
+            --label-regex unittest \
+            --output-on-failure \
+            --parallel "$(sysctl -n hw.ncpu)" \
+            --test-dir "cpp.build" \
+            --timeout 300
+      - name: Build example
+        run: |
+          cmake \
+            -S cpp/examples/minimal_build/ \
+            -B cpp/examples/minimal_build.build \
+            -GNinja \
+            -DCMAKE_INSTALL_PREFIX=$PWD/cpp.install
+          cmake --build cpp/examples/minimal_build.build
+          cd cpp/examples/minimal_build
+          ../minimal_build.build/arrow-example

.github/workflows/csharp.yml

@@ -62,7 +62,7 @@ jobs:
         with:
           python-version: 3
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Build
@@ -87,7 +87,7 @@ jobs:
         with:
           dotnet-version: ${{ matrix.dotnet }}
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Build
@@ -116,7 +116,7 @@ jobs:
         with:
           python-version: 3.12
       - name: Checkout Arrow
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Build
@@ -137,7 +137,7 @@ jobs:
     steps:
       - name: Checkout for utilities
         if: github.ref_type == 'tag'
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           path: arrow
       - name: Download source archive
@@ -155,7 +155,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
       - name: Checkout
         if: github.ref_type != 'tag'
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
       - name: Prepare version

.github/workflows/dev.yml

@@ -47,7 +47,7 @@ jobs:
     timeout-minutes: 15
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
       - name: Install pre-commit
@@ -86,7 +86,7 @@ jobs:
       GIT_COMMITTER_EMAIL: "github-actions[bot]@users.noreply.github.com"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
       - name: Install Python
@@ -107,10 +107,9 @@ jobs:
           gem install test-unit
           pip install "cython>=3" setuptools pytest requests setuptools-scm
       - name: Run Release Test
-        env:
-          ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         shell: bash
         run: |
+          echo "GH_TOKEN=${{ secrets.GITHUB_TOKEN }}" > dev/release/.env
           ci/scripts/release_test.sh $(pwd)
       - name: Run Merge Script Test
         shell: bash

.github/workflows/dev_pr.yml

@@ -43,7 +43,7 @@ jobs:
     name: Process
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: apache/arrow
           ref: main

.github/workflows/docs.yml

@@ -39,7 +39,7 @@ jobs:
       JDK: 17
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
       - name: Free up disk space

.github/workflows/docs_light.yml

@@ -49,7 +49,7 @@ jobs:
       PYTHON: "3.9"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
       - name: Cache Docker Volumes