Merge pull request #47 from trailofbits/build_improve

* Ninja instead of Makefiles for Linux on LLVM
* Enable ccache on LLVM [more work to do on this one to use cache on rebuilds]
* New supported LLVM targets: ARMv7 and NVPTX

Author: artemdinaburg

Dockerfile

@@ -11,7 +11,7 @@ ARG LLVM_VERSION
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
-    apt-get install -qqy python2.7 python3 python3-pip build-essential \
+    apt-get install -qqy ninja-build python2.7 python3 python3-pip build-essential ccache \
          liblzma-dev clang libssl-dev && \
     rm -rf /var/lib/apt/lists/*
 
@@ -32,32 +32,34 @@ RUN ./pkgman.py \
   rm -rf build && mkdir build && \
   rm -rf sources && mkdir sources
 
-RUN ./pkgman.py \
+RUN mkdir -p /cache && ./pkgman.py \
   --c_compiler=/usr/bin/clang \
   --cxx_compiler=/usr/bin/clang++ \
   --llvm_version=${LLVM_VERSION} \
   --verbose \
+  --use_ccache \
   --exclude_libcxx \
   "--additional_paths=${BOOTSTRAP}/cmake/bin" \
   "--repository_path=${LIBRARIES}" \
   "--packages=llvm" && \
   rm -rf build && mkdir build && \
-  rm -rf sources && mkdir sources
+  rm -rf sources && mkdir sources && rm -rf /cache
 
 FROM base as cxx-common-build
 
 WORKDIR /cxx-common
 ARG BOOTSTRAP
 ARG LIBRARIES
 
-RUN ./pkgman.py \
+RUN mkdir -p /cache && ./pkgman.py \
   --cxx_compiler="${LIBRARIES}/llvm/bin/clang++" \
   --c_compiler="${LIBRARIES}/llvm/bin/clang" \
+  --use_ccache \
   --verbose \
   "--additional_paths=${BOOTSTRAP}/cmake/bin:${LIBRARIES}/llvm/bin" \
   "--repository_path=${LIBRARIES}" \
   "--packages=cmake,capstone,google,xed,capnproto" && \
   rm -rf build && mkdir build && \
-  rm -rf sources && mkdir sources
+  rm -rf sources && mkdir sources && rm -rf /cache
 
 ENTRYPOINT ["/bin/bash"]

pkgman.py

@@ -34,6 +34,7 @@ def main():
   arg_parser.add_argument("--c_compiler", type=str, help="The C compiler to use.")
   arg_parser.add_argument("--exclude_libcxx", help="Exclude libc++ from the build", action='store_true')
   arg_parser.add_argument("--use_no_ssl_requests", help="Use the requests library to download files, and do so without SSL verification. If behind a firewall/proxy, this can help", action='store_true')
+  arg_parser.add_argument("--use_ccache", help="Enable ccache build for LLVM", action='store_true')
 
   default_repository_path = ""
   if get_platform_type() == "windows":
@@ -95,6 +96,8 @@ def main():
   properties["verbose"] = args.verbose
   properties["debug"] = args.debug
 
+  properties["ccache"] = args.use_ccache
+
   # Make sure that file downloading will work.
   if args.use_no_ssl_requests:
     properties["use_requests_for_downloading"] = True

pkgman/installers/common.py

@@ -575,7 +575,7 @@ def common_installer_llvm(properties):
 
   arch_list = "'X86"
   if sys.platform != "win32":
-    arch_list += ";AArch64;Sparc"
+    arch_list += ";AArch64;Sparc;NVPTX;ARM"
   arch_list += "'"
 
   cppstd = "11"
@@ -586,6 +586,15 @@ def common_installer_llvm(properties):
                                                                                            "-DLLVM_ENABLE_RTTI=ON", "-DLLVM_INCLUDE_EXAMPLES=OFF",
                                                                                            "-DLLVM_INCLUDE_TESTS=OFF"]
 
+  if properties["ccache"]:
+    print(" i Enabling ccache on /cache ... ")
+    # some versions of LLVM use CCACHE_MAX_SIZE, others use CCACHE_SIZE
+    cmake_command.extend(
+        ["-DLLVM_CCACHE_BUILD=ON",
+         "-DLLVM_CCACHE_SIZE=10G",
+         "-DLLVM_CCACHE_DIR=/cache",
+         "-DLLVM_CCACHE_MAXSIZE=10G"])
+
   if use_libcxx:
     if int(properties["llvm_version"]) < 371:
       cmake_command += ["-DLIBCXX_ENABLE_SHARED=NO"]

pkgman/installers/utils.py

@@ -59,6 +59,9 @@ def get_cmake_generator(use_clang=True):
     parameters = ["-G", "Visual Studio 15 2017 Win64"]
     if use_clang:
       parameters += ["-T", "LLVM-vs2014"]
+  elif sys.platform.startswith("linux"):
+    # do ninja-build on linux for faster builds
+    parameters = ["-G", "Ninja"]
 
   return parameters
 

travis.sh

@@ -39,7 +39,7 @@ linux_initialize() {
   fi
 
   printf " > Installing the required packages...\n"
-  sudo apt-get install -qqy python2.7 build-essential python3 python3-pip clang
+  sudo apt-get install -qqy python2.7 build-essential python3 python3-pip clang ninja-build
   if [ $? -ne 0 ] ; then
     printf " x Could not install the required dependencies\n"
     return 1