Add ci scripts (#1)

* Add ci scripts

* Move config files to ci/kokoro/docker

* Remove dump-logs etc

* Simplified grpc installation

* Add Ubuntu 16.04 kokoro config

* Address code review

Author: Takashi Matsuo

CMakeLists.txt

@@ -45,7 +45,7 @@ set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_SHA256
     "6b8a9b2bcb4476e9a5a9872869996f0d639c8d5df76dd8a893e79201f211b1cf")
 
 include(ExternalProject)
-externalproject_add(googleapis_download
+ExternalProject_Add(googleapis_download
                     EXCLUDE_FROM_ALL ON
                     PREFIX "${CMAKE_BINARY_DIR}/external/googleapis"
                     URL ${GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL}
@@ -54,7 +54,7 @@ externalproject_add(googleapis_download
                     BUILD_COMMAND ""
                     INSTALL_COMMAND ""
                     LOG_DOWNLOAD OFF)
-externalproject_get_property(googleapis_download SOURCE_DIR)
+ExternalProject_Get_Property(googleapis_download SOURCE_DIR)
 set(GOOGLEAPIS_CPP_SOURCE "${SOURCE_DIR}")
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
@@ -267,23 +267,21 @@ install(TARGETS ${googleapis_cpp_installed_libraries_list}
 
 foreach (target ${googleapis_cpp_installed_libraries_list})
     google_cloud_cpp_install_proto_library_headers("${target}")
-    google_cloud_cpp_install_proto_library_protos("${target}" "${GOOGLEAPIS_CPP_SOURCE}")
+    google_cloud_cpp_install_proto_library_protos("${target}"
+                                                  "${GOOGLEAPIS_CPP_SOURCE}")
 endforeach ()
 
 # Export the CMake targets to make it easy to create configuration files.
 install(EXPORT googleapis-targets
         DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/googleapis")
 
 # Setup global variables used in the following *.in files.
-set(
-    GOOGLE_CLOUD_CPP_CONFIG_VERSION_MAJOR ${GOOGLEAPIS_CPP_PROTOS_VERSION_MAJOR}
-    )
-set(
-    GOOGLE_CLOUD_CPP_CONFIG_VERSION_MINOR ${GOOGLEAPIS_CPP_PROTOS_VERSION_MINOR}
-    )
-set(
-    GOOGLE_CLOUD_CPP_CONFIG_VERSION_PATCH ${GOOGLEAPIS_CPP_PROTOS_VERSION_PATCH}
-    )
+set(GOOGLE_CLOUD_CPP_CONFIG_VERSION_MAJOR
+    ${GOOGLEAPIS_CPP_PROTOS_VERSION_MAJOR})
+set(GOOGLE_CLOUD_CPP_CONFIG_VERSION_MINOR
+    ${GOOGLEAPIS_CPP_PROTOS_VERSION_MINOR})
+set(GOOGLE_CLOUD_CPP_CONFIG_VERSION_PATCH
+    ${GOOGLEAPIS_CPP_PROTOS_VERSION_PATCH})
 
 # Use a function to create a scope for the variables.
 function (googleapis_cpp_install_pc target)
@@ -322,8 +320,8 @@ function (googleapis_cpp_install_pc target)
     string(CONCAT GOOGLE_CLOUD_CPP_PC_REQUIRES ${_target_pc_requires})
     set(GOOGLE_CLOUD_CPP_PC_LIBS "-l${target}")
     configure_file("cmake/config.pc.in" "${target}.pc" @ONLY)
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${target}.pc" DESTINATION
-                  "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${target}.pc"
+            DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 endfunction ()
 
 # Create and install the pkg-config files.
@@ -355,8 +353,8 @@ string(CONCAT GOOGLE_CLOUD_CPP_PC_REQUIRES
               " libcares")
 set(GOOGLE_CLOUD_CPP_PC_LIBS "")
 configure_file("cmake/config.pc.in" "googleapis.pc" @ONLY)
-install(FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis.pc" DESTINATION
-              "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis.pc"
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
 
 # Create and install the CMake configuration files.
 configure_file("cmake/config.cmake.in" "googleapis-config.cmake" @ONLY)
@@ -367,5 +365,4 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis-config.cmake"
               "${PROJECT_SOURCE_DIR}/cmake/FindgRPC.cmake"
               "${PROJECT_SOURCE_DIR}/cmake/FindProtobufTargets.cmake"
               "${PROJECT_SOURCE_DIR}/cmake/CompileProtos.cmake"
-              DESTINATION
-              "${CMAKE_INSTALL_LIBDIR}/cmake/googleapis")
+        DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/googleapis")

ci/check-style.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+if [[ "${CHECK_STYLE}" != "yes" ]]; then
+  echo "Skipping code style check as it is disabled for this build."
+  exit 0
+fi
+
+# This script assumes it is running the top-level google-cloud-cpp directory.
+
+readonly BINDIR="$(dirname "$0")"
+
+# Build paths to ignore in find(1) commands by reading .gitignore.
+declare -a ignore=( -path ./.git )
+if [[ -f .gitignore ]]; then
+  while read -r line; do
+    case "${line}" in
+    [^#]*/*) ignore+=( -o -path "./$(expr "${line}" : '\(.*\)/')" ) ;;
+    [^#]*)   ignore+=( -o -name "${line}" ) ;;
+    esac
+  done < .gitignore
+fi
+
+replace_original_if_changed() {
+  if [[ $# != 2 ]]; then
+    return 1
+  fi
+
+  local original="$1"
+  local reformatted="$2"
+
+  if cmp -s "${original}" "${reformatted}"; then
+    rm -f "${reformatted}"
+  else
+    chmod --reference="${original}" "${reformatted}"
+    mv -f "${reformatted}" "${original}"
+  fi
+}
+
+# Apply cmake_format to all the CMake list files.
+#     https://github.com/cheshirekow/cmake_format
+find . \( "${ignore[@]}" \) -prune -o \
+       \( -name 'CMakeLists.txt' -o -name '*.cmake' \) \
+       -print0 |
+  while IFS= read -r -d $'\0' file; do
+    cmake-format "${file}" >"${file}.tmp"
+    replace_original_if_changed "${file}" "${file}.tmp"
+  done
+
+# Apply buildifier to fix the BUILD and .bzl formatting rules.
+#    https://github.com/bazelbuild/buildtools/tree/master/buildifier
+find . \( "${ignore[@]}" \) -prune -o \
+       \( -name BUILD -o -name '*.bzl' \) \
+       -print0 |
+  xargs -0 buildifier -mode=fix
+
+# Apply shellcheck(1) to emit warnings for common scripting mistakes.
+find . \( "${ignore[@]}" \) -prune -o \
+       -iname '*.sh' -exec shellcheck \
+         --exclude=SC1090 \
+         --exclude=SC2034 \
+         --exclude=SC2153 \
+         --exclude=SC2181 \
+       '{}' \;
+
+# Report any differences created by running the formatting tools.
+git diff --ignore-submodules=all --color --exit-code .

ci/colors.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -z "${COLOR_RESET+x}" ]; then
+  if type tput >/dev/null 2>&1; then
+    readonly COLOR_RED="$(tput setaf 1)"
+    readonly COLOR_GREEN="$(tput setaf 2)"
+    readonly COLOR_YELLOW="$(tput setaf 3)"
+    readonly COLOR_RESET="$(tput sgr0)"
+  else
+    readonly COLOR_RED=""
+    readonly COLOR_GREEN=""
+    readonly COLOR_YELLOW=""
+    readonly COLOR_RESET=""
+  fi
+fi

ci/install-grpc.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+mkdir -p /var/tmp/Downloads
+cd /var/tmp/Downloads
+
+# Install protobuf
+wget -q https://github.com/google/protobuf/archive/v3.8.0.tar.gz
+tar -xf v3.8.0.tar.gz
+(cd protobuf-3.8.0/cmake;
+ cmake \
+   -DCMAKE_BUILD_TYPE=Release \
+   -DBUILD_SHARED_LIBS=yes \
+   -Dprotobuf_BUILD_TESTS=OFF \
+   -H. -Bcmake-out
+ cmake --build cmake-out --target install -- -j "$(nproc)"
+ ldconfig
+)
+
+# Install grpc
+wget -q https://github.com/grpc/grpc/archive/v1.22.0.tar.gz
+tar -xf v1.22.0.tar.gz
+(cd grpc-1.22.0;
+ make -j "$(nproc)"
+ make install
+ ldconfig
+)

ci/kokoro/Dockerfile.centos

@@ -0,0 +1,51 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO_VERSION=7
+FROM centos:${DISTRO_VERSION}
+
+# Add /usr/local/lib
+
+# Search paths tweak for the build
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib64/pkgconfig
+ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64
+ENV PATH=/usr/local/bin:${PATH}
+
+# First install the development tools and OpenSSL. The development tools
+# distributed with CentOS (notably CMake) are too old to build
+# `google-cloud-cpp`. In these instructions, we use `cmake3` obtained from
+# [Software Collections](https://www.softwarecollections.org/).
+
+RUN rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+RUN yum install -y centos-release-scl
+RUN yum-config-manager --enable rhel-server-rhscl-7-rpms
+RUN yum makecache && \
+    yum install -y automake cmake3 curl-devel gcc gcc-c++ git libtool \
+        make openssl-devel pkgconfig tar wget which zlib-devel
+RUN ln -sf /usr/bin/cmake3 /usr/bin/cmake && ln -sf /usr/bin/ctest3 /usr/bin/ctest
+
+# Install c-ares
+RUN mkdir -p /var/tmp/Downloads; \
+    cd /var/tmp/Downloads; \
+    wget -q https://github.com/c-ares/c-ares/archive/cares-1_15_0.tar.gz; \
+    tar -xf cares-1_15_0.tar.gz; \
+    cd /var/tmp/Downloads/c-ares-cares-1_15_0; \
+    ./buildconf && ./configure && make -j $(nproc); \
+    make install; \
+    ldconfig
+
+# Install grpc from source
+WORKDIR /var/tmp/ci
+COPY install-grpc.sh /var/tmp/ci
+RUN /var/tmp/ci/install-grpc.sh

ci/kokoro/Dockerfile.fedora-install

@@ -0,0 +1,38 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO_VERSION=30
+FROM fedora:${DISTRO_VERSION}
+
+# Fedora includes packages for gRPC, libcurl, and OpenSSL that are recent enough
+# for `google-cloud-cpp`. Install these packages and additional development
+# tools to compile the dependencies:
+RUN dnf makecache && \
+    dnf install -y clang clang-tools-extra cmake doxygen findutils gcc-c++ git \
+        grpc-devel grpc-plugins libcxx-devel libcxxabi-devel libcurl-devel \
+        make openssl-devel pkgconfig protobuf-compiler python-pip ShellCheck \
+        tar wget zlib-devel
+
+# Install the the buildifier tool, which does not compile with the default
+# golang compiler for Ubuntu 16.04 and Ubuntu 18.04.
+RUN wget -q -O /usr/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/0.17.2/buildifier
+RUN chmod 755 /usr/bin/buildifier
+
+# Install cmake_format to automatically format the CMake list files.
+#     https://github.com/cheshirekow/cmake_format
+# Pin this to an specific version because the formatting changes when the
+# "latest" version is updated, and we do not want the builds to break just
+# because some third party changed something.
+RUN pip install --upgrade pip
+RUN pip install numpy cmake_format==0.5.2