Use pre-built libraries to build wheels on Linux

Author: BewareMyPower

.github/workflows/ci-build-release-wheels.yaml

@@ -57,9 +57,6 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
-      - uses: docker/setup-buildx-action@v2
-      - run: build-support/copy-deps-versionfile.sh
-
       - name: Build Manylinux Docker image
         uses: docker/build-push-action@v3
         with:

.github/workflows/ci-pr-validation.yaml

@@ -126,9 +126,6 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
-      - uses: docker/setup-buildx-action@v2
-      - run: build-support/copy-deps-versionfile.sh
-
       - name: Build Manylinux Docker image
         uses: docker/build-push-action@v3
         with:

build-support/copy-deps-versionfile.sh

@@ -21,46 +21,39 @@
 set -e -x
 
 cd /pulsar-client-python
+source build-support/dep-url.sh
 
 # Build cpp wheels
-if [[ $ARCH == "aarch64" ]]; then
-    export VCPKG_FORCE_SYSTEM_BINARIES=1
-fi
-PULSAR_CPP_VERSION=$(dep-version.py pulsar-cpp)
-. /dep-url.sh
-download_dependency /dependencies.yaml pulsar-cpp
-cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}
-
-git clone https://github.com/microsoft/vcpkg.git
-cd vcpkg
-
-# manylinux2014 does not have ninja in the system package manager
-git clone https://github.com/ninja-build/ninja.git
-cd ninja
-git checkout release
-./configure.py --bootstrap
-mv ninja /usr/bin/
-cd ..
-./bootstrap-vcpkg.sh
-cd ..
+PULSAR_CPP_VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}')
 
-cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON
-cmake --build build -j8 --target install
-cd ..
-rm -rf apache-pulsar-client-cpp-$(PULSAR_CPP_VERSION)
+if [ $CPP_BINARY_TYPE == "rpm" ]; then
+    if [ $ARCH == "aarch64" ]; then
+        RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-arm64/aarch64
+    else
+        RPM_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/rpm-x86_64/x86_64
+    fi
+    curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-1.$ARCH.rpm
+    curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-devel-$PULSAR_CPP_VERSION-1.$ARCH.rpm
+    curl -O -L $RPM_ROOT_DIR/apache-pulsar-client-debuginfo-$PULSAR_CPP_VERSION-1.$ARCH.rpm
+    rpm -ivh *.rpm
+else # apk
+    if [ $ARCH == "aarch64" ]; then
+        APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-arm64/aarch64
+    else
+        APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-x86_64/x86_64
+    fi
+    curl -O -L $APK_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-r0.apk
+    curl -O -L $APK_ROOT_DIR/apache-pulsar-client-dev-$PULSAR_CPP_VERSION-r0.apk
+    apk add --allow-untrusted *.apk
+fi
 
-PYBIND11_VERSION=$(./build-support/dep-version.py pybind11)
-curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
-tar zxf v${PYBIND11_VERSION}.tar.gz
+download_dependency $PWD/dependencies.yaml pybind11
 rm -rf pybind11
-mv pybind11-${PYBIND11_VERSION} pybind11
-
-rm -f CMakeCache.txt CMakeFiles
-
-cmake . \
-      -DCMAKE_BUILD_TYPE=Release
+mv pybind11-* pybind11
 
-make -j4
+cmake -B build -DCMAKE_BUILD_TYPE=Release
+cmake --build build -j8
+mv build/lib_pulsar.so .
 
 ./setup.py bdist_wheel
 

pkg/build-wheel-inside-docker.sh

@@ -24,20 +24,11 @@ ARG PYTHON_VERSION
 ARG PYTHON_SPEC
 ARG ARCH
 
+ENV CPP_BINARY_TYPE=rpm
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 ENV PYTHON_SPEC=${PYTHON_SPEC}
 
 ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
 ENV ARCH=${ARCH}
 
-ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include
-ENV PYTHON_LIBRARIES   /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
-ENV Python3_ROOT_DIR /opt/python/${PYTHON_SPEC}/
-
-RUN pip3 install pyyaml setuptools
-
-ADD .build/dependencies.yaml /
-ADD .build/dep-version.py /usr/local/bin
-ADD .build/dep-url.sh /
-
-RUN yum install -y curl zip unzip tar perl-IPC-Cmd
+RUN pip3 install setuptools

pkg/manylinux2014/Dockerfile

@@ -30,13 +30,6 @@ ARG ARCH
 ENV ARCH=${ARCH}
 
 ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}"
-ENV PYTHON_INCLUDE_DIR /opt/python/${PYTHON_SPEC}/include
-ENV PYTHON_LIBRARIES   /opt/python/${PYTHON_SPEC}/lib/python${PYTHON_VERSION}
 
-RUN pip install pyyaml setuptools
-
-ADD .build/dependencies.yaml /
-ADD .build/dep-version.py /usr/local/bin
-ADD .build/dep-url.sh /
-
-RUN apk add build-base cmake ninja zip unzip curl git
+RUN pip install setuptools
+ENV CPP_BINARY_TYPE="apk"