ci(gcb): add demo-centos-7 install build (#6239)

devjgm · web-flow · commit 1d7d8285348f · 2021-04-13T12:59:03.000-04:00
* ci(gcb): add demo-centos-7 install build Part of #6163 This is the first example of how I plan to move what we currently call "install" builds (defined in `ci/kokoro/install`) over to GCB where I plan to call them "demo" builds. The idea is that we'll have a single script called `demo-install.sh` (in this PR) that installs `google-cloud-cpp`, and then tests the install by running the quickstarts. We'll then have a number of `demo-<os>.Dockerfile` files (`demo-centos-7.Dockerfile` is included in this PR), and a number of trigger files, which associate a `demo-<os>` distro with the one `demo-install.sh` script. In subsequent PRs, I'll add more `demo-<os>.Dockerfile`s, and more triggers for the other platforms. Separately, I'll also update the packaging scripts to use these new files for generating the `packaging.md` file. Note about naming: I thought that "install" was a slightly ambiguous term for these builds, and thought "demo" was a little more descriptive since we use these builds to demonstrate how to install the deps and install our library on various platforms. Happy to change the naming back to "install" if others dislike the new naming. * removed unneeded export
diff --git a/ci/cloudbuild/builds/demo-install.sh b/ci/cloudbuild/builds/demo-install.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This build script serves two main purposes:
+#
+# 1. It demonstrates to users how to install `google-cloud-cpp`. We will
+#    extract part of this script into user-facing markdown documentation.
+# 2. It verifies that the installed artifacts work by compiling and running the
+#    quickstart programs against the installed artifacts.
+
+set -eu
+
+source "$(dirname "$0")/../../lib/init.sh"
+source module ci/cloudbuild/builds/lib/cmake.sh
+source module ci/etc/quickstart-config.sh
+source module ci/lib/io.sh
+
+## [START packaging.md]
+
+# Pick a location to install the artifacts, e.g., `/usr/local` or `/opt`
+PREFIX="${HOME}/google-cloud-cpp-installed"
+cmake -DBUILD_TESTING=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -H. -Bcmake-out
+cmake --build cmake-out -- -j "$(nproc)"
+cmake --build cmake-out --target install
+
+## [END packaging.md]
+
+# Tells pkg-config where the artifacts are, so the Makefile builds work.
+export PKG_CONFIG_PATH="${PREFIX}/lib64/pkgconfig:${PKG_CONFIG_PATH:-}"
+
+# Verify that the installed artifacts are usable by running the quickstarts.
+for lib in $(quickstart::libraries); do
+  mapfile -t run_args < <(quickstart::arguments "${lib}")
+  io::log_h2 "Building quickstart: ${lib}"
+  if [[ "${PROJECT_ID:-}" != "cloud-cpp-testing-resources" ]]; then
+    run_args=() # Empties these args so we don't execute quickstarts below
+    io::log_yellow "Not executing quickstarts," \
+      "which can only run in GCB project 'cloud-cpp-testing-resources'"
+  fi
+
+  io::log "[ CMake ]"
+  src_dir="${PROJECT_ROOT}/google/cloud/${lib}/quickstart"
+  bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-${lib}"
+  cmake -H"${src_dir}" -B"${bin_dir}" "-DCMAKE_PREFIX_PATH=${PREFIX}"
+  cmake --build "${bin_dir}"
+  test "${#run_args[@]}" -eq 0 || "${bin_dir}/quickstart" "${run_args[@]}"
+
+  echo
+  io::log "[ Make ]"
+  make -C "${src_dir}"
+  test "${#run_args[@]}" -eq 0 || "${src_dir}/quickstart" "${run_args[@]}"
+done
diff --git a/ci/cloudbuild/demo-centos-7.Dockerfile b/ci/cloudbuild/demo-centos-7.Dockerfile
@@ -0,0 +1,194 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO_VERSION=7
+FROM centos:${DISTRO_VERSION} AS devtools
+ARG NCPU=4
+
+## [START packaging.md]
+
+# First install the development tools and OpenSSL. The development tools
+# distributed with CentOS 7 are too old to build the project. In these
+# instructions, we use `cmake3` and `gcc-7` obtained from
+# [Software Collections](https://www.softwarecollections.org/).
+
+# ```bash
+RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum install -y centos-release-scl yum-utils
+RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
+RUN yum makecache && \
+    yum install -y automake ccache cmake3 curl-devel devtoolset-7 gcc gcc-c++ \
+        git libtool make openssl-devel pkgconfig re2-devel tar wget which \
+        zlib-devel
+RUN ln -sf /usr/bin/cmake3 /usr/bin/cmake && ln -sf /usr/bin/ctest3 /usr/bin/ctest
+# ```
+
+## [START IGNORED]
+# In order to use the `devtoolset-7` Software Collection we're supposed to run
+# `scl enable devtoolset-7 bash`, which starts a new shell with the environment
+# configured correctly. However, we can't do that in this Dockerfile, AND we
+# want the instructions that we generate for the user to say the right thing.
+# So this block is ignored, and we manually set some environment variables to
+# make the devtoolset-7 available. After this ignored block, we'll include the
+# correct instructions for the user. NOTE: These env values were obtained by
+# manually running the `scl ...` command (above) then copying the values set in
+# its environment.
+ENV PATH /opt/rh/devtoolset-7/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV LD_LIBRARY_PATH /opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib:/opt/rh/devtoolset-7/root/usr/lib64/dyninst:/opt/rh/devtoolset-7/root/usr/lib/dyninst:/opt/rh/devtoolset-7/root/usr/lib64:/opt/rh/devtoolset-7/root/usr/lib
+## [END IGNORED]
+# Start a bash shell with its environment configured to use the tools installed
+# by `devtoolset-7`.
+# **IMPORTANT**: All the following commands should be run from this new shell.
+# ```bash
+# scl enable devtoolset-7 bash
+# ```
+
+# The following steps will install libraries and tools in `/usr/local`. By
+# default CentOS-7 does not search for shared libraries in these directories,
+# there are multiple ways to solve this problem, the following steps are one
+# solution:
+
+# ```bash
+RUN (echo "/usr/local/lib" ; echo "/usr/local/lib64") | \
+    tee /etc/ld.so.conf.d/usrlocal.conf
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig
+ENV PATH=/usr/local/bin:${PATH}
+# ```
+
+# #### Abseil
+
+# We need a recent version of Abseil.
+
+# ```bash
+WORKDIR /var/tmp/build
+RUN wget -q https://github.com/abseil/abseil-cpp/archive/20200923.3.tar.gz && \
+    tar -xf 20200923.3.tar.gz && \
+    cd abseil-cpp-20200923.3 && \
+    sed -i 's/^#define ABSL_OPTION_USE_\(.*\) 2/#define ABSL_OPTION_USE_\1 0/' "absl/base/options.h" && \
+    cmake \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_TESTING=OFF \
+      -DBUILD_SHARED_LIBS=yes \
+      -DCMAKE_CXX_STANDARD=11 \
+      -H. -Bcmake-out && \
+    cmake --build cmake-out -- -j ${NCPU:-4} && \
+    cmake --build cmake-out --target install -- -j ${NCPU:-4} && \
+    ldconfig
+# ```
+
+# #### Protobuf
+
+# We need to install a version of Protobuf that is recent enough to support the
+# Google Cloud Platform proto files:
+
+# ```bash
+WORKDIR /var/tmp/build
+RUN wget -q https://github.com/google/protobuf/archive/v3.14.0.tar.gz && \
+    tar -xf v3.14.0.tar.gz && \
+    cd protobuf-3.14.0/cmake && \
+    cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_SHARED_LIBS=yes \
+        -Dprotobuf_BUILD_TESTS=OFF \
+        -H. -Bcmake-out && \
+    cmake --build cmake-out -- -j ${NCPU:-4} && \
+    cmake --build cmake-out --target install -- -j ${NCPU:-4} && \
+    ldconfig
+# ```
+
+# #### c-ares
+
+# Recent versions of gRPC require c-ares >= 1.11, while CentOS-7
+# distributes c-ares-1.10. Manually install a newer version:
+
+# ```bash
+WORKDIR /var/tmp/build
+RUN wget -q https://github.com/c-ares/c-ares/archive/cares-1_14_0.tar.gz && \
+    tar -xf cares-1_14_0.tar.gz && \
+    cd c-ares-cares-1_14_0 && \
+    ./buildconf && ./configure && make -j ${NCPU:-4} && \
+    make install && \
+    ldconfig
+# ```
+
+# #### gRPC
+
+# We also need a version of gRPC that is recent enough to support the Google
+# Cloud Platform proto files. We manually install it using:
+
+# ```bash
+WORKDIR /var/tmp/build
+RUN wget -q https://github.com/grpc/grpc/archive/v1.35.0.tar.gz && \
+    tar -xf v1.35.0.tar.gz && \
+    cd grpc-1.35.0 && \
+    cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DgRPC_INSTALL=ON \
+        -DgRPC_BUILD_TESTS=OFF \
+        -DgRPC_ABSL_PROVIDER=package \
+        -DgRPC_CARES_PROVIDER=package \
+        -DgRPC_PROTOBUF_PROVIDER=package \
+        -DgRPC_RE2_PROVIDER=package \
+        -DgRPC_SSL_PROVIDER=package \
+        -DgRPC_ZLIB_PROVIDER=package \
+        -H. -Bcmake-out && \
+    cmake --build cmake-out -- -j ${NCPU:-4} && \
+    cmake --build cmake-out --target install -- -j ${NCPU:-4} && \
+    ldconfig
+# ```
+
+# #### crc32c
+
+# The project depends on the Crc32c library, we need to compile this from
+# source:
+
+# ```bash
+WORKDIR /var/tmp/build
+RUN wget -q https://github.com/google/crc32c/archive/1.1.0.tar.gz && \
+    tar -xf 1.1.0.tar.gz && \
+    cd crc32c-1.1.0 && \
+    cmake \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_SHARED_LIBS=yes \
+        -DCRC32C_BUILD_TESTS=OFF \
+        -DCRC32C_BUILD_BENCHMARKS=OFF \
+        -DCRC32C_USE_GLOG=OFF \
+        -H. -Bcmake-out && \
+    cmake --build cmake-out -- -j ${NCPU:-4} && \
+    cmake --build cmake-out --target install -- -j ${NCPU:-4} && \
+    ldconfig
+# ```
+
+# #### nlohmann_json library
+
+# The project depends on the nlohmann_json library. We use CMake to
+# install it as this installs the necessary CMake configuration files.
+# Note that this is a header-only library, and often installed manually.
+# This leaves your environment without support for CMake pkg-config.
+
+# ```bash
+WORKDIR /var/tmp/build
+RUN wget -q https://github.com/nlohmann/json/archive/v3.9.0.tar.gz && \
+    tar -xzf v3.9.0.tar.gz && \
+    cd json-3.9.0 && \
+    cmake \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DBUILD_SHARED_LIBS=yes \
+      -DBUILD_TESTING=OFF \
+      -H. -Bcmake-out/nlohmann/json && \
+    cmake --build cmake-out/nlohmann/json --target install -- -j ${NCPU} && \
+    ldconfig
+# ```
+
+## [END packaging.md]
diff --git a/ci/cloudbuild/triggers/demo-centos-7-ci.yaml b/ci/cloudbuild/triggers/demo-centos-7-ci.yaml
@@ -0,0 +1,13 @@
+filename: ci/cloudbuild/cloudbuild.yaml
+github:
+  name: google-cloud-cpp
+  owner: googleapis
+  push:
+    branch: ^master$
+name: demo-centos-7-ci
+substitutions:
+  _BUILD_NAME: demo-install
+  _DISTRO: demo-centos-7
+  _TRIGGER_TYPE: ci
+tags:
+- ci
diff --git a/ci/cloudbuild/triggers/demo-centos-7-pr.yaml b/ci/cloudbuild/triggers/demo-centos-7-pr.yaml
@@ -0,0 +1,14 @@
+filename: ci/cloudbuild/cloudbuild.yaml
+github:
+  name: google-cloud-cpp
+  owner: googleapis
+  pullRequest:
+    branch: ^master$
+    commentControl: COMMENTS_ENABLED_FOR_EXTERNAL_CONTRIBUTORS_ONLY
+name: demo-centos-7-pr
+substitutions:
+  _BUILD_NAME: demo-install
+  _DISTRO: demo-centos-7
+  _TRIGGER_TYPE: pr
+tags:
+- pr
