apacheGH-36411: [Python] Use scikit-build-core as build backend for PyArrow and get rid of setup.py (apache#49259)

### Rationale for this change

Move our PyArrow build backend from setuptools and a custom setup.py to scikit-build-core which is just build backend for CMake related projects.

### What changes are included in this PR?

Move from setuptools to scikit-build-core and remove PyArrow setup.py. Update some of the build requirements and minor fixes.
A custom build backend has been also been created in order to wrap scikit-build-core in order to fix problems on License files for monorepos.
pyproject.toml metadata validation expects license files to exist before exercising the build backend that's why we create symlinks. Our thin build backend will just make those symlinks hard-links in order for license and notice files to contain the contents and be added as part of the sdist.

Remove flags that are not used anymore (were only part of setup.py) and documented and validated how the same flags have to be used now.

### Are these changes tested?

Yes all Python CI tests, wheels and sdist are successful.

### Are there any user-facing changes?

Yes, users building PyArrow will now require the new build dependencies to exercise the build and depending on the flags used they might require to use the new documented way of using those flags.

* GitHub Issue: apache#36411

Lead-authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Co-authored-by: Joris Van den Bossche <jorisvandenbossche@gmail.com>
Co-authored-by: Rok Mihevc <rok@mihevc.org>
Signed-off-by: Raúl Cumplido <raulcumplido@gmail.com>

Author: 3 people

.env

@@ -99,8 +99,8 @@ VCPKG="66c0373dc7fca549e5803087b9487edfe3aca0a1"    # 2026.01.16 Release
 # ci/docker/python-*-windows-*.dockerfile or the vcpkg config.
 # This is a workaround for our CI problem that "archery docker build" doesn't
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-02-25
-PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-02-25
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2026-03-04
+PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2026-03-04
 
 # Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
 # See https://github.com/conan-io/conan-docker-tools#readme and

.github/workflows/dev.yml

@@ -103,7 +103,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit openssl
-          pip install "cython>=3.1" setuptools pytest requests setuptools-scm
+          pip install build "cython>=3.1" pytest requests scikit-build-core setuptools-scm
       - name: Run Release Test
         shell: bash
         run: |

ci/conda_env_python.txt

@@ -29,5 +29,5 @@ numpy>=1.16.6
 pytest
 pytest-faulthandler
 s3fs>=2023.10.0
-setuptools>=77
+scikit-build-core
 setuptools_scm>=8

ci/scripts/python_build.bat

@@ -111,10 +111,7 @@ echo "=== CCACHE Stats after build ==="
 ccache -sv
 
 echo "=== Building Python ==="
-set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
-set PYARROW_BUILD_VERBOSE=1
 set PYARROW_BUNDLE_ARROW_CPP=ON
-set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
 set PYARROW_WITH_ACERO=%ARROW_ACERO%
 set PYARROW_WITH_AZURE=%ARROW_AZURE%
 set PYARROW_WITH_DATASET=%ARROW_DATASET%
@@ -137,6 +134,6 @@ pushd %SOURCE_DIR%\python
 %PYTHON_CMD% -m pip install -r requirements-build.txt || exit /B 1
 
 @REM Build PyArrow
-%PYTHON_CMD% -m pip install --no-deps --no-build-isolation -vv . || exit /B 1
+%PYTHON_CMD% -m pip install --no-deps --no-build-isolation -vv -C build.verbose=true . || exit /B 1
 
 popd

ci/scripts/python_build.sh

@@ -59,9 +59,8 @@ if [ -n "${CONDA_PREFIX}" ]; then
   conda list
 fi
 
-export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
-export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
-
+export CMAKE_BUILD_PARALLEL_LEVEL=${n_jobs}
+export CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
 export PYARROW_WITH_ACERO=${ARROW_ACERO:-OFF}
 export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
 export PYARROW_WITH_CUDA=${ARROW_CUDA:-OFF}
@@ -76,8 +75,6 @@ export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION:-ON}
 export PYARROW_WITH_S3=${ARROW_S3:-OFF}
 export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF}
 
-export PYARROW_PARALLEL=${n_jobs}
-
 : "${CMAKE_PREFIX_PATH:=${ARROW_HOME}}"
 export CMAKE_PREFIX_PATH
 export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
@@ -90,11 +87,9 @@ export DYLD_LIBRARY_PATH=${ARROW_HOME}/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_P
 rm -rf "${python_build_dir}"
 cp -aL "${source_dir}" "${python_build_dir}"
 pushd "${python_build_dir}"
-# - Cannot call setup.py as it may install in the wrong directory
-#   on Debian/Ubuntu (ARROW-15243).
 # - Cannot use build isolation as we want to use specific dependency versions
 #   (e.g. Numpy, Pandas) on some CI jobs.
-${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv .
+${PYTHON:-python} -m pip install --no-deps --no-build-isolation -vv -C cmake.build-type="${CMAKE_BUILD_TYPE:-Debug}" .
 popd
 
 if [ "${BUILD_DOCS_PYTHON}" == "ON" ]; then

ci/scripts/python_sdist_build.sh

@@ -23,5 +23,5 @@ source_dir=${1}/python
 
 pushd "${source_dir}"
 export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
-${PYTHON:-python} setup.py sdist
+${PYTHON:-python} -m build --sdist
 popd

ci/scripts/python_sdist_test.sh

@@ -25,8 +25,7 @@ export ARROW_SOURCE_DIR=${arrow_dir}
 export ARROW_TEST_DATA=${arrow_dir}/testing/data
 export PARQUET_TEST_DATA=${arrow_dir}/cpp/submodules/parquet-testing/data
 
-export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
-export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE:-debug}
+export CMAKE_GENERATOR=${CMAKE_GENERATOR:-Ninja}
 export PYARROW_WITH_ACERO=${ARROW_ACERO:-ON}
 export PYARROW_WITH_AZURE=${ARROW_AZURE:-OFF}
 export PYARROW_WITH_S3=${ARROW_S3:-OFF}
@@ -64,7 +63,7 @@ if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
   . "${ARROW_PYTHON_VENV}/bin/activate"
 fi
 
-${PYTHON:-python} -m pip install "${sdist}"
+${PYTHON:-python} -m pip install -C cmake.build-type="${CMAKE_BUILD_TYPE:-Debug}" "${sdist}"
 
 # shellcheck disable=SC2086
 pytest -r s ${PYTEST_ARGS:-} --pyargs pyarrow

ci/scripts/python_wheel_macos_build.sh

@@ -146,9 +146,7 @@ cmake --build . --target install
 popd
 
 echo "=== (${PYTHON_VERSION}) Building wheel ==="
-export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-export PYARROW_BUNDLE_ARROW_CPP=1
-export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
+export PYARROW_BUNDLE_ARROW_CPP=ON
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
 export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}
@@ -161,15 +159,18 @@ export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
 export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
 export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
 export PYARROW_WITH_S3=${ARROW_S3}
-export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"
 export ARROW_HOME=${build_dir}/install
 # PyArrow build configuration
 export CMAKE_PREFIX_PATH=${build_dir}/install
 # Set PyArrow version explicitly
 export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION}
 
 pushd ${source_dir}/python
-python -m build --sdist --wheel . --no-isolation
+python -m build --sdist --wheel . --no-isolation \
+  -C build.verbose=true \
+  -C cmake.build-type=${CMAKE_BUILD_TYPE:-Debug} \
+  -C cmake.args="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}" \
+  -C cmake.args="-DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"
 popd
 
 echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ==="

ci/scripts/python_wheel_windows_build.bat

@@ -115,11 +115,7 @@ cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1
 popd
 
 echo "=== (%PYTHON%) Building wheel ==="
-set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
-set PYARROW_BUILD_VERBOSE=1
 set PYARROW_BUNDLE_ARROW_CPP=ON
-set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
-set PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%"
 set PYARROW_WITH_ACERO=%ARROW_ACERO%
 set PYARROW_WITH_AZURE=%ARROW_AZURE%
 set PYARROW_WITH_DATASET=%ARROW_DATASET%
@@ -138,7 +134,10 @@ set CMAKE_PREFIX_PATH=C:\arrow-dist
 pushd C:\arrow\python
 
 @REM Build wheel
-%PYTHON_CMD% -m build --sdist --wheel . --no-isolation || exit /B 1
+%PYTHON_CMD% -m build --sdist --wheel . --no-isolation -vv ^
+    -C build.verbose=true ^
+    -C cmake.build-type=%CMAKE_BUILD_TYPE% ^
+    -C cmake.args="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%" || exit /B 1
 
 @REM Repair the wheel with delvewheel
 @REM

ci/scripts/python_wheel_xlinux_build.sh

@@ -154,10 +154,7 @@ popd
 check_arrow_visibility
 
 echo "=== (${PYTHON_VERSION}) Building wheel ==="
-export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-export PYARROW_BUNDLE_ARROW_CPP=1
-export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
-export PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION}"
+export PYARROW_BUNDLE_ARROW_CPP=ON
 export PYARROW_WITH_ACERO=${ARROW_ACERO}
 export PYARROW_WITH_AZURE=${ARROW_AZURE}
 export PYARROW_WITH_DATASET=${ARROW_DATASET}
@@ -175,7 +172,10 @@ export ARROW_HOME=/tmp/arrow-dist
 export CMAKE_PREFIX_PATH=/tmp/arrow-dist
 
 pushd /arrow/python
-python -m build --sdist --wheel . --no-isolation
+python -m build --sdist --wheel . --no-isolation \
+    -C build.verbose=true \
+    -C cmake.build-type=${CMAKE_BUILD_TYPE:-Debug} \
+    -C cmake.args="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION}"
 
 echo "=== Strip symbols from wheel ==="
 mkdir -p dist/temp-fix-wheel