Use vcpkg to build debian packages (#506)

Author: BewareMyPower

.github/workflows/ci-build-binary-artifacts.yaml

@@ -56,7 +56,6 @@ jobs:
         run: build-support/generate-source-archive.sh
 
       - uses: docker/setup-buildx-action@v2
-      - run: build-support/copy-deps-versionfile.sh
 
       - name: Build dependencies Docker image
         uses: docker/build-push-action@v3

.github/workflows/ci-pr-validation.yaml

@@ -326,7 +326,6 @@ jobs:
         run: build-support/generate-source-archive.sh
 
       - uses: docker/setup-buildx-action@v2
-      - run: build-support/copy-deps-versionfile.sh
 
       - name: Build dependencies Docker image
         uses: docker/build-push-action@v3

build-support/copy-deps-versionfile.sh

@@ -23,104 +23,15 @@ FROM debian:11
 
 ARG PLATFORM
 
-# perl is required to install OpenSSL
 RUN apt-get update -y && \
      apt-get install -y \
         curl \
         g++ \
         make \
-        perl \
         dpkg-dev \
-        python3 \
-        python3-pip \
-        git
-
-RUN pip3 install pyyaml
-
-ADD .build/dependencies.yaml /
-ADD .build/dep-version.py /usr/local/bin
-
-# Download and compile boost
-RUN BOOST_VERSION=$(dep-version.py boost) && \
-    curl -O -L https://github.com/boostorg/boost/releases/download/boost-${BOOST_VERSION}/boost-${BOOST_VERSION}.tar.gz && \
-    tar zxf boost-${BOOST_VERSION}.tar.gz && \
-    cd boost-${BOOST_VERSION} && \
-    ./bootstrap.sh --with-libraries=regex && \
-    ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \
-    rm -rf /boost-${BOOST_VERSION}.tar.gz /boost-${BOOST_VERSION}
-
-RUN CMAKE_VERSION=$(dep-version.py cmake) && \
-    curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \
-    tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \
-    cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \
-    cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \
-    rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz
-
-# Download and compile protobuf
-RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
-    curl -O -L  https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    cd protobuf-${PROTOBUF_VERSION}/ && \
-    CXXFLAGS=-fPIC ./configure && \
-    make -j8 && make install && ldconfig && \
-    rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}
-
-# ZLib
-RUN ZLIB_VERSION=$(dep-version.py zlib) && \
-    curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
-    tar xfz v${ZLIB_VERSION}.tar.gz && \
-    cd zlib-${ZLIB_VERSION} && \
-    CFLAGS="-fPIC -O3" ./configure && \
-    make -j8 && make install && \
-    rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
-
-# Zstandard
-RUN ZSTD_VERSION=$(dep-version.py zstd) && \
-    curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
-    tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
-    cd zstd-${ZSTD_VERSION} && \
-    CFLAGS="-fPIC -O3" make -j8 && \
-    make install && \
-    rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
-
-# Snappy
-RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
-    curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
-    tar xfz ${SNAPPY_VERSION}.tar.gz && \
-    cd snappy-${SNAPPY_VERSION} && \
-    CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
-    make -j8 && make install && \
-    rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
-
-# Roaring
-RUN ROARING_VERSION=$(dep-version.py roaring) && \
-    curl -O -L  https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v${ROARING_VERSION}.tar.gz && \
-    tar xfz v${ROARING_VERSION}.tar.gz && \
-    cd CRoaring-${ROARING_VERSION} && \
-    mkdir build && cd build && CXXFLAGS="-fPIC -O3" cmake .. && \
-    make -j8 && make install && \
-    rm -rf /v${ROARING_VERSION}.tar.gz /CRoaring-${ROARING_VERSION}
-
-RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
-    OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
-    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
-    ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \
-    make -j8 && make install && \
-    rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}
-
-ENV LD_LIBRARY_PATH /usr/local/ssl/lib/:
-ENV OPENSSL_ROOT_DIR /usr/local/ssl/
-
-# LibCurl
-RUN CURL_VERSION=$(dep-version.py curl) && \
-    CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
-    curl -O -L  https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
-    tar xfz curl-${CURL_VERSION}.tar.gz && \
-    cd curl-${CURL_VERSION} && \
-    CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd --without-libpsl && \
-    make -j8 && make install && \
-    rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
-
-
+        git \
+        zip \
+        unzip \
+        tar \
+        pkg-config \
+        ninja-build

build-support/dep-version.py

@@ -20,6 +20,10 @@
 
 set -e -x
 
+if [[ $# -gt 0 ]]; then
+    CMAKE_ARCH=$1
+fi
+
 cd /pulsar-client-cpp
 SRC_ROOT_DIR=$(pwd)
 cd pkg/deb
@@ -37,12 +41,28 @@ cd BUILD
 tar xfz $SRC_ROOT_DIR/apache-pulsar-client-cpp-$POM_VERSION.tar.gz
 pushd $CPP_DIR
 
-# link libraries for protoc
-export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+git clone https://github.com/microsoft/vcpkg.git
+# The cmake from debian:11 is 3.18, while vcpkg needs cmake >= 3.21 to build
+if [[ $CMAKE_ARCH ]]; then
+    export CMAKE_VERSION=3.31.9
+    curl -O -L https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION-linux-$CMAKE_ARCH.tar.gz
+    export PATH=$PWD/cmake-$CMAKE_VERSION-linux-$CMAKE_ARCH/bin/:$PATH
+    tar zxf cmake-*.tar.gz
+fi
+cmake --version
 
 chmod +x $(find . -name "*.sh")
-cmake . -DBUILD_TESTS=OFF -DBUILD_PERF_TOOLS=OFF -DLINK_STATIC=ON
-make -j 3
+if [[ $CMAKE_ARCH == "aarch64" ]]; then
+    export VCPKG_FORCE_SYSTEM_BINARIES=1
+fi
+cmake -B build -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON
+cmake --build build -j8
+./build-support/merge_archives_vcpkg.sh $PWD/build
+
+cp build/lib/libpulsar.a lib/libpulsar.a
+cp build/lib/libpulsar.so lib/libpulsar.so
+cp build/libpulsarwithdeps.a lib/libpulsarwithdeps.a
 popd
 
 DEST_DIR=apache-pulsar-client

dependencies.yaml

@@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64}
 docker run -v $ROOT_DIR:/pulsar-client-cpp \
         --env PLATFORM=arm64 \
         $IMAGE_NAME \
-        /pulsar-client-cpp/pkg/deb/build-deb.sh
+        /pulsar-client-cpp/pkg/deb/build-deb.sh \
+        aarch64

pkg/deb/Dockerfile

@@ -27,4 +27,5 @@ IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64}
 docker run -v $ROOT_DIR:/pulsar-client-cpp \
         --env PLATFORM=amd64 \
         $IMAGE_NAME \
-        /pulsar-client-cpp/pkg/deb/build-deb.sh
+        /pulsar-client-cpp/pkg/deb/build-deb.sh \
+        x86_64

pkg/deb/build-deb.sh

@@ -30,16 +30,9 @@ RUN yum update -y && \
         rpm-build \
         which \
         createrepo \
-        git \
-        python3 \
-        python3-pip
+        git
 RUN dnf --enablerepo=powertools install -y libstdc++-static
 
-RUN pip3 install pyyaml
-
-ADD .build/dependencies.yaml /
-ADD .build/dep-version.py /usr/local/bin
-
 # Vcpkg does not provide pre-built binaries for Arm architectures so we need to build vcpkg from source
 RUN yum install -y cmake
 RUN dnf --enablerepo=devel install -y ninja-build