build: update vcpkg version (and hence dependencies) (apache#3542)

lidavidm · kou · web-flow · commit ad2b150e0b3c · 2025-10-09T18:17:05.000+09:00
- Update to recent vcpkg - Remove patches that are no longer necessary - Use our own Docker image to build wheels instead of relying on upstream Closes apache#3516. --------- Co-authored-by: Sutou Kouhei <kou@cozmixng.org>
diff --git a/.env b/.env
@@ -32,7 +32,7 @@ JDK=21
 MANYLINUX=2014
 MAVEN=3.9.10
 PLATFORM=linux/amd64
-PYTHON=3.9
+PYTHON=3.13
 GO=1.24.1
 ARROW_MAJOR_VERSION=18
 DOTNET=8.0
@@ -41,7 +41,7 @@ DOTNET=8.0
 # ci/scripts/install_vcpkg.sh script. Keep in sync with apache/arrow .env.
 # When updating, also update the docs, which list the version of libpq/SQLite
 # that vcpkg (and hence our wheels) ship
-VCPKG="f7423ee180c4b7f40d43402c2feb3859161ef625"    # 2024.06.15 Release
+VCPKG="4334d8b4c8916018600212ab4dd4bbdc343065d1"    # 2025.09.17 Release
 
 # These are used to tell tests where to find services for integration testing.
 # They are valid if the services are started with the compose config.
diff --git a/ci/docker/python-wheel-manylinux.dockerfile b/ci/docker/python-wheel-manylinux.dockerfile
@@ -15,24 +15,66 @@
 # specific language governing permissions and limitations
 # under the License.
 
-ARG ARCH
 ARG MANYLINUX
+FROM quay.io/pypa/manylinux${MANYLINUX}:latest
+
+ARG CMAKE=4.1.2
+ARG GO
+ARG NINJA=1.13.1
 ARG PYTHON
-ARG REPO
 ARG VCPKG
+ARG TARGETPLATFORM
 
-FROM ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
-
-ARG ARCH
-ARG GO
+SHELL ["/bin/bash", "-i", "-c"]
+ENTRYPOINT ["/bin/bash", "-i", "-c"]
 
+# -------------------- System Dependencies --------------------
+# Some of these dependencies are needed to build things like OpenSSL in vcpkg
+RUN ulimit -n 1024 && yum install -y autoconf curl git flex perl-IPC-Cmd unzip wget yum-utils zip
 # docker is aliased to podman by AlmaLinux, but we want real Docker
 # (podman is just too different)
-RUN yum remove -y docker ; yum install -y yum-utils
+RUN ulimit -n 1024 && yum remove -y docker
 RUN yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
-RUN yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
-# arm64v8 -> arm64
-RUN wget --no-verbose https://go.dev/dl/go${GO}.linux-${ARCH/v8/}.tar.gz && \
-    tar -C /usr/local -xzf go${GO}.linux-${ARCH/v8/}.tar.gz && \
-    rm go${GO}.linux-${ARCH/v8/}.tar.gz
+RUN ulimit -n 1024 && yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+# -------------------- Python --------------------
+RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON/./}-cp${PYTHON/./}) && \
+    echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
+ENV PATH="/opt/python/cp${PYTHON/./}-cp${PYTHON/./}/bin:${PATH}"
+
+# -------------------- CMake, Go --------------------
+RUN mkdir -p /.cache/go-build                         \
+    && chmod 777 /.cache/go-build                     \
+    && git config --global --add safe.directory /adbc \
+    && cp /root/.gitconfig /.gitconfig                \
+    && chmod 777 /.gitconfig
+
+RUN if [[ ${TARGETPLATFORM} == "linux/amd64" ]]; then                                                           \
+        export ARCH="amd64" CMAKE_ARCH=x86_64;                                                                  \
+    elif [[ ${TARGETPLATFORM} == "linux/arm64" ]]; then                                                         \
+        export ARCH="arm64" CMAKE_ARCH=aarch64;                                                                 \
+    else                                                                                                        \
+        echo "Unsupported platform: ${TARGETPLATFORM}";                                                         \
+        exit 1;                                                                                                 \
+    fi &&                                                                                                       \
+    wget --no-verbose -O cmake.tar.gz                                                                           \
+      https://github.com/Kitware/CMake/releases/download/v${CMAKE}/cmake-${CMAKE}-linux-${CMAKE_ARCH}.tar.gz && \
+    wget --no-verbose -O go.tar.gz https://go.dev/dl/go${GO}.linux-${ARCH}.tar.gz &&                            \
+    tar -C /usr/local -xzf cmake.tar.gz &&                                                                      \
+    tar -C /usr/local -xzf go.tar.gz &&                                                                         \
+    rm -f cmake.tar.gz go.tar.gz
+
 ENV PATH="/usr/local/go/bin:${PATH}"
+
+# -------------------- Ninja --------------------
+RUN mkdir -p /tmp/ninja &&                                                                   \
+    wget --no-verbose -O - "https://github.com/ninja-build/ninja/archive/v${NINJA}.tar.gz" | \
+      tar -xzf - --directory /tmp/ninja --strip-components=1 &&                              \
+    pushd /tmp/ninja &&                                                                      \
+    ./configure.py --bootstrap &&                                                            \
+    mv ninja "/usr/local/bin" &&                                                             \
+    rm -rf /tmp/ninja
+
+# -------------------- vcpkg --------------------
+ADD ci/scripts/install_vcpkg.sh /
+RUN /install_vcpkg.sh /opt/vcpkg ${VCPKG} && rm -f /install_vcpkg.sh
diff --git a/ci/scripts/install_vcpkg.sh b/ci/scripts/install_vcpkg.sh
@@ -35,7 +35,7 @@ vcpkg_version=${2:-$default_vcpkg_version}
 vcpkg_ports_patch=${3:-$default_vcpkg_ports_patch}
 
 # reduce the fetched data using a shallow clone
-git clone --shallow-since=2024-01-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
+git clone --shallow-since=2025-09-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}
 
 pushd ${vcpkg_destination}
 
diff --git a/ci/scripts/python_util.sh b/ci/scripts/python_util.sh
@@ -70,33 +70,15 @@ function build_drivers {
 
     echo "=== Setup VCPKG ==="
 
-    pushd "${VCPKG_ROOT}"
-    # XXX: work around lz4 CMakeLists being out of date
-    # https://github.com/lz4/lz4/issues/1550
-    patch -N -p1 < "${source_dir}/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch"
-
-    # XXX: make vcpkg retry downloads https://github.com/microsoft/vcpkg/discussions/20583
-    patch -N -p1 < "${source_dir}/ci/vcpkg/0002-Retry-downloads.patch"
-
-    # XXX: backport fix for CMake 4 and macOS
-    patch -N -p1 < "${source_dir}/ci/vcpkg/0003-Fix-CMake-4-OSX.patch"
-    popd
-
     # Need to install sqlite3 to make CMake be able to find it below
-    "${VCPKG_ROOT}/vcpkg" install sqlite3 \
-          --overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
-          --triplet "${VCPKG_DEFAULT_TRIPLET}"
-
-    "${VCPKG_ROOT}/vcpkg" install libpq \
+    "${VCPKG_ROOT}/vcpkg" install libpq sqlite3 \
           --overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
           --triplet "${VCPKG_DEFAULT_TRIPLET}"
 
     echo "=== Building drivers ==="
     mkdir -p ${build_dir}
     pushd ${build_dir}
     cmake \
-        -DADBC_BUILD_SHARED=ON \
-        -DADBC_BUILD_STATIC=ON \
         -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
         -DCMAKE_INSTALL_LIBDIR=lib \
         -DCMAKE_INSTALL_PREFIX=${build_dir} \
@@ -105,6 +87,8 @@ function build_drivers {
         ${CMAKE_ARGUMENTS} \
         -DVCPKG_OVERLAY_TRIPLETS="${VCPKG_OVERLAY_TRIPLETS}" \
         -DVCPKG_TARGET_TRIPLET="${VCPKG_DEFAULT_TRIPLET}" \
+        -DADBC_BUILD_SHARED=ON \
+        -DADBC_BUILD_STATIC=ON \
         -DADBC_DRIVER_BIGQUERY=ON \
         -DADBC_DRIVER_FLIGHTSQL=ON \
         -DADBC_DRIVER_MANAGER=ON \
diff --git a/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch b/ci/vcpkg/0001-Work-around-lz4-CMake-https-github.com-lz4-lz4-issue.patch
diff --git a/ci/vcpkg/0002-Retry-downloads.patch b/ci/vcpkg/0002-Retry-downloads.patch
diff --git a/ci/vcpkg/0003-Fix-CMake-4-OSX.patch b/ci/vcpkg/0003-Fix-CMake-4-OSX.patch
diff --git a/compose.yaml b/compose.yaml
@@ -171,32 +171,31 @@ services:
   # relocate on a separate image so that we can use a newer docker for cibuildwheel
 
   python-wheel-manylinux-build:
-    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
+    image: ghcr.io/apache/arrow-adbc:manylinux${MANYLINUX}-go${GO}-python${PYTHON}-vcpkg${VCPKG}
     build:
       context: .
-      cache_from:
-        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
       dockerfile: ci/docker/python-wheel-manylinux.dockerfile
       args:
-        ARCH: ${ARCH}
         GO: ${GO}
         MANYLINUX: ${MANYLINUX}
         PYTHON: ${PYTHON}
-        REPO: ${REPO}
         VCPKG: ${VCPKG}
+      platforms:
+        - linux/amd64
+        - linux/arm64
     volumes:
       - .:/adbc
     # Must set safe.directory so go/miniver won't error when calling git
     command: "'git config --global --add safe.directory /adbc && /adbc/ci/scripts/python_wheel_unix_build.sh ${ARCH} /adbc /adbc/build'"
 
   python-wheel-manylinux-relocate:
-    image: ${REPO}:adbc-python-${PYTHON}-wheel-relocate
-    platform: ${PLATFORM}
+    image: ghcr.io/apache/arrow-adbc:python${PYTHON}
     build:
       context: .
-      cache_from:
-        - ${REPO}:adbc-python-${PYTHON}-wheel-relocate
       dockerfile: ci/docker/python-wheel-manylinux-relocate.dockerfile
+      platforms:
+        - linux/amd64
+        - linux/arm64
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - .:/adbc
diff --git a/docs/source/driver/postgresql.rst b/docs/source/driver/postgresql.rst
@@ -315,7 +315,7 @@ Software Versions
 =================
 
 For Python wheels, the shipped version of the PostgreSQL client libraries is
-15.2.  For conda-forge packages, the version of libpq is the same as the
+16.9.  For conda-forge packages, the version of libpq is the same as the
 version of libpq in your Conda environment.
 
 The PostgreSQL driver is tested against PostgreSQL versions 11 through 16.
diff --git a/docs/source/driver/sqlite.rst b/docs/source/driver/sqlite.rst
@@ -242,6 +242,6 @@ Driver-specific options:
 Software Versions
 =================
 
-For Python wheels, the shipped version of SQLite is 3.40.1.  For conda-forge
+For Python wheels, the shipped version of SQLite is 3.50.4.  For conda-forge
 packages, the version of sqlite is the same as the version of sqlite in your
 Conda environment.
