Merge remote-tracking branch 'origin/main' into feat/flight-sql-client

Author: HackPoint

.github/workflows/csharp.yml

@@ -191,7 +191,7 @@ jobs:
             dev/release/utils-generate-checksum.sh "${artifact}"
           done
       - name: Upload
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: nuget
           path: |

.github/workflows/pr_review_trigger.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Upload PR review Payload"
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           path: "${{ github.event_path }}"
           name: "pr_review_payload"

.github/workflows/r.yml

@@ -177,7 +177,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: test-output-${{ matrix.ubuntu }}-${{ matrix.r }}
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -237,7 +237,7 @@ jobs:
         if: always()
       - name: Save the test output
         if: always()
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: test-output-bundled
           path: r/check/arrow.Rcheck/tests/testthat.Rout*
@@ -299,7 +299,7 @@ jobs:
         # So that they're unique when multiple are downloaded in the next step
         shell: bash
         run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
-      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
           path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip
@@ -337,7 +337,7 @@ jobs:
           echo "$HOME/.local/bin" >> $GITHUB_PATH
       - run: mkdir r/windows
       - name: Download artifacts
-        uses: actions/download-artifact@v4.1.9
+        uses: actions/download-artifact@v4.2.1
         with:
           name: libarrow-rtools40-ucrt64.zip
           path: r/windows

.pre-commit-config.yaml

@@ -195,9 +195,27 @@ repos:
           ?^ci/scripts/c_glib_build\.sh$|
           ?^ci/scripts/c_glib_test\.sh$|
           ?^c_glib/test/run-test\.sh$|
+          ?^dev/release/05-binary-upload\.sh$|
           ?^dev/release/07-binary-verify\.sh$|
+          ?^dev/release/post-03-binary\.sh$|
+          ?^dev/release/post-11-python\.sh$|
           ?^dev/release/utils-generate-checksum\.sh$|
           )
+  - repo: https://github.com/scop/pre-commit-shfmt
+    rev: v3.11.0-1
+    hooks:
+      - id: shfmt
+        args:
+          # The default args is "--write --simplify" but we don't use
+          # "--simplify". Because it's conflicted will ShellCheck.
+          - "--write"
+        # TODO: Remove this when we fix all lint failures
+        files: >-
+          (
+          ?^dev/release/05-binary-upload\.sh$|
+          ?^dev/release/post-03-binary\.sh$|
+          ?^dev/release/post-11-python\.sh$|
+          )
   - repo: https://github.com/trim21/pre-commit-mirror-meson
     rev: v1.6.1
     hooks:

.shellcheckrc

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+external-sources=true
+source-path=SCRIPTDIR

c_glib/test/flight-sql/test-client.rb

@@ -23,6 +23,7 @@ def setup
     @server = nil
     omit("Arrow Flight SQL is required") unless defined?(ArrowFlightSQL)
     omit("Unstable on Windows") if Gem.win_platform?
+    omit("Unstable on x86_64 macOS") if /x86_64-darwin/.match?(RUBY_PLATFORM)
     @server = Helper::FlightSQLServer.new
     host = "127.0.0.1"
     location = ArrowFlight::Location.new("grpc://#{host}:0")

c_glib/test/flight/test-client.rb

@@ -22,6 +22,7 @@ def setup
     @server = nil
     omit("Arrow Flight is required") unless defined?(ArrowFlight)
     omit("Unstable on Windows") if Gem.win_platform?
+    omit("Unstable on x86_64 macOS") if /x86_64-darwin/.match?(RUBY_PLATFORM)
     require_gi_bindings(3, 4, 7)
     @server = Helper::FlightServer.new
     host = "127.0.0.1"

c_glib/test/flight/test-stream-reader.rb

@@ -22,6 +22,7 @@ def setup
     @server = nil
     omit("Arrow Flight is required") unless defined?(ArrowFlight)
     omit("Unstable on Windows") if Gem.win_platform?
+    omit("Unstable on x86_64 macOS") if /x86_64-darwin/.match?(RUBY_PLATFORM)
     require_gi_bindings(3, 4, 5)
     @server = Helper::FlightServer.new
     host = "127.0.0.1"

ci/docker/python-free-threaded-wheel-musllinux-test-imports.dockerfile

@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+RUN apk add --no-cache \
+    bash \
+    build-base \
+    bzip2-dev \
+    g++ \
+    git \
+    libffi-dev \
+    libnsl-dev \
+    libtirpc-dev \
+    linux-headers \
+    ncurses-dev \
+    openssl-dev \
+    pkgconf \
+    tzdata \
+    zlib-dev
+
+# Install Python3.13.2 without GIL
+RUN wget https://github.com/python/cpython/archive/refs/tags/v3.13.2.tar.gz && \
+    tar -xzf v3.13.2.tar.gz && \
+    rm v3.13.2.tar.gz && \
+    cd cpython-3.13.2/ && \
+    ./configure --disable-gil --with-ensurepip && \
+    make -j && \
+    make install && \
+    cd ../ && \
+    rm -rf cpython-3.13.2/
+
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+
+ENV PYTHON_GIL 0
+ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"

ci/docker/python-free-threaded-wheel-musllinux-test-unittests.dockerfile

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+RUN apk add --no-cache \
+    bash \
+    build-base \
+    bzip2-dev \
+    g++ \
+    git \
+    libffi-dev \
+    libnsl-dev \
+    libtirpc-dev \
+    linux-headers \
+    ncurses-dev \
+    openssl-dev \
+    pkgconf \
+    tzdata \
+    zlib-dev
+
+# Install Python3.13.2 without GIL
+RUN wget https://github.com/python/cpython/archive/refs/tags/v3.13.2.tar.gz && \
+    tar -xzf v3.13.2.tar.gz && \
+    rm v3.13.2.tar.gz && \
+    cd cpython-3.13.2/ && \
+    ./configure --disable-gil --with-ensurepip && \
+    make -j && \
+    make install && \
+    cd ../ && \
+    rm -rf cpython-3.13.2/
+
+ENV ARROW_PYTHON_VENV /arrow-dev
+RUN python3.13t -m venv ${ARROW_PYTHON_VENV}
+
+ENV PYTHON_GIL 0
+ENV PATH "${ARROW_PYTHON_VENV}/bin:${PATH}"
+
+ENV TZDIR=/usr/share/zoneinfo
+RUN cp /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+# pandas doesn't provide wheels for aarch64 yet, so we have to install nightly Cython
+# along with the rest of pandas' build dependencies and disable build isolation
+COPY python/requirements-wheel-test.txt /arrow/python/
+RUN python -m pip install \
+    --pre \
+    --prefer-binary \
+    --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \
+    Cython numpy
+RUN python -m pip install "meson-python==0.13.1" "meson==1.2.1" wheel "versioneer[toml]" ninja
+RUN python -m pip install --no-build-isolation -r /arrow/python/requirements-wheel-test.txt