Simplify dependency upgrade process and upgrade Pulsar C++ client to 3.1.0 (#56)

BewareMyPower · web-flow · commit 724012b1019a · 2022-12-08T21:15:22.000+08:00
### Motivation Currently there are many files that require the download URLs of dependencies to download them. It's hard to maintain if some download URL changed. For example, #17 updates the ZLib download URL under https://zlib.net/fossils/ for macOS build. However, the ZLib download URL for Linux is under https://github.com/madler/zlib/archive/. The same goes for the Pulsar C++ client, it's hard to test another URL because the candidates and the official releases are stored in different paths. ### Modifications Add a `dep-url.sh` to provide two shell functions: - `pulsar_cpp_base_url`: Print the base URL of the Pulsar C++ client release, there are the source code or binaries in the subpath. - `download_dependency`: Download the source code according to the dependency file and the dependency name. Then apply the `dep-url.sh` in all files that need to download the source or binary of the dependencies. In addition, this PR upgrades the `pulsar-cpp` dependency to 3.1.0 so that the Windows build can depend on an official release.
diff --git a/.github/workflows/ci-build-release-wheels.yaml b/.github/workflows/ci-build-release-wheels.yaml
@@ -167,10 +167,11 @@ jobs:
       - name: Download Pulsar C++ client on Windows
         shell: bash
         run: |
+          source ./build-support/dep-url.sh
+          BASE_URL=$(pulsar_cpp_base_url $(grep pulsar-cpp dependencies.yaml | awk '{print $2}'))
           mkdir -p ${{ env.PULSAR_CPP_DIR }}
           cd ${{ env.PULSAR_CPP_DIR }}
-          # TODO: switch to official releases
-          curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp/pulsar-client-cpp-3.1.0-candidate-1/x64-windows-static.tar.gz
+          curl -O -L ${BASE_URL}/x64-windows-static.tar.gz
           tar zxf x64-windows-static.tar.gz
           mv x64-windows-static/* .
           ls -l ${{ env.PULSAR_CPP_DIR }}
diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml
@@ -199,10 +199,11 @@ jobs:
       - name: Download Pulsar C++ client on Windows
         shell: bash
         run: |
+          source ./build-support/dep-url.sh
+          BASE_URL=$(pulsar_cpp_base_url $(grep pulsar-cpp dependencies.yaml | awk '{print $2}'))
           mkdir -p ${{ env.PULSAR_CPP_DIR }}
           cd ${{ env.PULSAR_CPP_DIR }}
-          # TODO: switch to official releases
-          curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp/pulsar-client-cpp-3.1.0-candidate-1/x64-windows-static.tar.gz
+          curl -O -L ${BASE_URL}/x64-windows-static.tar.gz
           tar zxf x64-windows-static.tar.gz
           mv x64-windows-static/* .
           ls -l ${{ env.PULSAR_CPP_DIR }}
diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh
@@ -26,4 +26,5 @@ for dir in manylinux2014 manylinux_musl; do
   mkdir -p pkg/$dir/.build
   cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build
   cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build
+  cp $ROOT_DIR/build-support/dep-url.sh pkg/$dir/.build
 done
diff --git a/build-support/dep-url.sh b/build-support/dep-url.sh
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+pulsar_cpp_base_url() {
+  if [[ $# -ne 1 ]]; then
+    echo "Usage: pulsar_cpp_base_url <version>"
+    exit 1
+  fi
+  VERSION=$1
+  # TODO: use official release url from https://archive.apache.org/
+  echo "https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp/pulsar-client-cpp-${VERSION}-candidate-3"
+}
+
+download_dependency() {
+  if [[ $# -ne 2 ]]; then
+    echo "Usage: download_dependency <dependency-name> <dependency-version>"
+    exit 1
+  fi
+
+  DEP_FILE=$1
+  DEP=$2
+  # Here we don't use read command to make it available in Alpine
+  VERSION=$(grep $DEP $DEP_FILE | sed 's/://' | awk '{print $2}')
+
+  case $DEP in
+    "cmake")
+      URL=https://github.com/Kitware/CMake/releases/download/v${VERSION}/cmake-${VERSION}-linux-${ARCH}.tar.gz
+      ;;
+    "pulsar-cpp")
+      URL=$(pulsar_cpp_base_url $VERSION)/apache-pulsar-client-cpp-${VERSION}.tar.gz
+      ;;
+    "pybind11")
+      URL=https://github.com/pybind/pybind11/archive/refs/tags/v${VERSION}.tar.gz
+      ;;
+    "boost")
+      VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g')
+      URL=https://boostorg.jfrog.io/artifactory/main/release/${VERSION}/source/boost_${VERSION_UNDERSCORE}.tar.gz
+      ;;
+    "protobuf")
+      URL=https://github.com/google/protobuf/releases/download/v${VERSION}/protobuf-cpp-${VERSION}.tar.gz
+      ;;
+    "zlib")
+      URL=https://github.com/madler/zlib/archive/v${VERSION}.tar.gz
+      ;;
+    "zstd")
+      URL=https://github.com/facebook/zstd/releases/download/v${VERSION}/zstd-${VERSION}.tar.gz
+      ;;
+    "snappy")
+      URL=https://github.com/google/snappy/archive/refs/tags/${VERSION}.tar.gz
+      ;;
+    "openssl")
+      URL=https://github.com/openssl/openssl/archive/OpenSSL_$(echo $VERSION | sed 's/\./_/g').tar.gz
+      ;;
+    "curl")
+      VERSION_UNDERSCORE=$(echo $VERSION | sed 's/\./_/g')
+      URL=https://github.com/curl/curl/releases/download/curl-${VERSION_UNDERSCORE}/curl-${VERSION}.tar.gz
+      ;;
+    *)
+      echo "Unknown dependency $DEP for version $VERSION"
+      exit 1
+  esac
+  curl -O -L $URL
+  tar zxf $(basename $URL)
+}
diff --git a/build-support/install-dependencies.sh b/build-support/install-dependencies.sh
@@ -24,6 +24,7 @@ cd `dirname $0`
 
 CPP_CLIENT_VERSION=$(./dep-version.py pulsar-cpp ../dependencies.yaml)
 PYBIND11_VERSION=$(./dep-version.py pybind11 ../dependencies.yaml)
+source ./dep-url.sh
 
 if [ $USER != "root" ]; then
   SUDO="sudo"
@@ -35,7 +36,7 @@ export $(cat /etc/*-release | grep "^ID=")
 cd /tmp
 
 # Fetch the client binaries
-BASE_URL=https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-cpp-${CPP_CLIENT_VERSION}
+BASE_URL=$(pulsar_cpp_base_url ${CPP_CLIENT_VERSION})
 
 UNAME_ARCH=$(uname -m)
 if [ $UNAME_ARCH == 'aarch64' ]; then
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -18,7 +18,7 @@
 #
 
 cmake: 3.24.2
-pulsar-cpp: 3.0.0
+pulsar-cpp: 3.1.0
 pybind11: 2.10.1
 boost: 1.80.0
 protobuf: 3.20.0
diff --git a/pkg/mac/build-dependencies.sh b/pkg/mac/build-dependencies.sh
@@ -27,6 +27,7 @@ PYTHON_VERSION=$1
 PYTHON_VERSION_LONG=$2
 
 source pkg/mac/common.sh
+source build-support/dep-url.sh
 
 pip3 install pyyaml
 
@@ -49,8 +50,7 @@ PREFIX=$CACHE_DIR/install
 
 ###############################################################################
 if [ ! -f pybind11/.done ]; then
-    curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz
-    tar zxf v${PYBIND11_VERSION}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml pybind11
     mkdir -p $PREFIX/include/
     cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/
     mkdir -p pybind11
@@ -60,8 +60,7 @@ fi
 ###############################################################################
 if [ ! -f zlib-${ZLIB_VERSION}/.done ]; then
     echo "Building ZLib"
-    curl -O -L https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz
-    tar xfz zlib-$ZLIB_VERSION.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml zlib
     pushd zlib-$ZLIB_VERSION
       CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" ./configure --prefix=$PREFIX
       make -j16
@@ -102,7 +101,7 @@ fi
 OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
 if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then
     echo "Building OpenSSL"
-    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml openssl
     # -arch arm64 -arch x86_64
     tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz
 
@@ -140,8 +139,7 @@ fi
 BOOST_VERSION_=${BOOST_VERSION//./_}
 if [ ! -f boost/.done ]; then
     echo "Building Boost for Py $PYTHON_VERSION"
-    curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_}.tar.gz
-    tar xfz boost_${BOOST_VERSION_}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml boost
     cp -rf boost_${BOOST_VERSION_}/boost $PREFIX/include/
     mkdir -p boost
     touch .done
@@ -150,8 +148,7 @@ fi
 ###############################################################################
 if [ ! -f protobuf-${PROTOBUF_VERSION}/.done ]; then
     echo "Building Protobuf"
-    curl -O -L  https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
-    tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml protobuf
     pushd protobuf-${PROTOBUF_VERSION}
       CXXFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
             ./configure --prefix=$PREFIX
@@ -166,8 +163,7 @@ fi
 ###############################################################################
 if [ ! -f zstd-${ZSTD_VERSION}/.done ]; then
     echo "Building ZStd"
-    curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
-    tar xfz zstd-${ZSTD_VERSION}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml zstd
     pushd zstd-${ZSTD_VERSION}
       CFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" PREFIX=$PREFIX \
             make -j16 -C lib install-static install-includes
@@ -180,8 +176,7 @@ fi
 ###############################################################################
 if [ ! -f snappy-${SNAPPY_VERSION}/.done ]; then
     echo "Building Snappy"
-    curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz
-    tar xfz ${SNAPPY_VERSION}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml snappy
     pushd snappy-${SNAPPY_VERSION}
       CXXFLAGS="-fPIC -O3 -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
           cmake . -DCMAKE_INSTALL_PREFIX=$PREFIX -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF
@@ -197,8 +192,7 @@ fi
 if [ ! -f curl-${CURL_VERSION}/.done ]; then
     echo "Building LibCurl"
     CURL_VERSION_=${CURL_VERSION//./_}
-    curl -O -L  https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_}/curl-${CURL_VERSION}.tar.gz
-    tar xfz curl-${CURL_VERSION}.tar.gz
+    download_dependency $ROOT_DIR/dependencies.yaml curl
     pushd curl-${CURL_VERSION}
       CFLAGS="-fPIC -arch arm64 -arch x86_64 -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \
             ./configure --with-ssl=$PREFIX \
diff --git a/pkg/mac/build-pulsar-cpp.sh b/pkg/mac/build-pulsar-cpp.sh
@@ -24,6 +24,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)
 cd "${ROOT_DIR}"
 
 source pkg/mac/common.sh
+source build-support/dep-url.sh
 
 PULSAR_CPP_VERSION=$(./build-support/dep-version.py pulsar-cpp)
 
@@ -36,8 +37,7 @@ PREFIX=$CACHE_DIR_CPP_CLIENT/install
 DEPS_PREFIX=${CACHE_DIR_DEPS}/install
 
 ###############################################################################
-
-curl -O -L https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz
+download_dependency $ROOT_DIR/dependencies.yaml pulsar-cpp
 tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz
 
 if [ ! -f apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}/.done ]; then
diff --git a/pkg/manylinux2014/Dockerfile b/pkg/manylinux2014/Dockerfile
@@ -36,62 +36,56 @@ RUN pip3 install pyyaml
 
 ADD .build/dependencies.yaml /
 ADD .build/dep-version.py /usr/local/bin
+ADD .build/dep-url.sh /
 
 # Download and install boost
 RUN BOOST_VERSION=$(dep-version.py boost) && \
     BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \
-    curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \
-    tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml boost && \
     cp -r boost_${BOOST_VERSION_UNDESRSCORE}/boost /usr/include/ && \
     rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE}
 
 RUN CMAKE_VERSION=$(dep-version.py cmake) && \
-    curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz && \
-    tar xfz cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz && \
+    . /dep-url.sh && ARCH=${ARCH} download_dependency /dependencies.yaml cmake && \
     cp cmake-${CMAKE_VERSION}-linux-${ARCH}/bin/* /usr/bin/ && \
     cp -r cmake-${CMAKE_VERSION}-linux-${ARCH}/share/cmake-* /usr/share/ && \
     rm -rf cmake-${CMAKE_VERSION}-linux-${ARCH} cmake-${CMAKE_VERSION}-linux-${ARCH}.tar.gz
 
 # Download and compile protobuf
 RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \
-    curl -O -L  https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
-    tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml protobuf && \
     cd protobuf-${PROTOBUF_VERSION}/ && \
     CXXFLAGS=-fPIC ./configure && \
     make -j8 && make install && ldconfig && \
     rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION}
 
 # ZLib
 RUN ZLIB_VERSION=$(dep-version.py zlib) && \
-    curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \
-    tar xfz v${ZLIB_VERSION}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml zlib && \
     cd zlib-${ZLIB_VERSION} && \
     CFLAGS="-fPIC -O3" ./configure && \
     make -j8 && make install && \
     rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION}
 
 # Zstandard
 RUN ZSTD_VERSION=$(dep-version.py zstd) && \
-    curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \
-    tar xfz zstd-${ZSTD_VERSION}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml zstd && \
     cd zstd-${ZSTD_VERSION} && \
     CFLAGS="-fPIC -O3" make -j8 && \
     make install && \
     rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz
 
 # Snappy
 RUN SNAPPY_VERSION=$(dep-version.py snappy) && \
-    curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \
-    tar xfz ${SNAPPY_VERSION}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml snappy && \
     cd snappy-${SNAPPY_VERSION} && \
     CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \
     make -j8 && make install && \
     rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz
 
 RUN OPENSSL_VERSION=$(dep-version.py openssl) && \
     OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \
-    curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
-    tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml openssl && \
     cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \
     ./config -fPIC --prefix=/usr/local/ssl/ && \
     make -j8 && make install && \
@@ -102,18 +96,15 @@ ENV OPENSSL_ROOT_DIR /usr/local/ssl/
 
 # LibCurl
 RUN CURL_VERSION=$(dep-version.py curl) && \
-    CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \
-    curl -O -L  https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \
-    tar xfz curl-${CURL_VERSION}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml curl && \
     cd curl-${CURL_VERSION} && \
     CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \
     make -j8 && make install && \
     rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION}
 
 # Pulsar client C++
 RUN PULSAR_CPP_VERSION=$(dep-version.py pulsar-cpp) && \
-    curl -O -L https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
-    tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz && \
+    . /dep-url.sh && download_dependency /dependencies.yaml pulsar-cpp && \
     cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} && \
     cmake . -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_STATIC_LIB=OFF -DLINK_STATIC=ON && \
     make -j8 && \
diff --git a/pkg/manylinux_musl/Dockerfile b/pkg/manylinux_musl/Dockerfile

Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@`
`18`	`18`	`#`
`19`	`19`
`20`	`20`	`cmake: 3.24.2`
`21`		`-pulsar-cpp: 3.0.0`
	`21`	`+pulsar-cpp: 3.1.0`
`22`	`22`	`pybind11: 2.10.1`
`23`	`23`	`boost: 1.80.0`
`24`	`24`	`protobuf: 3.20.0`