Skip to content

Commit a513e20

Browse files
author
Peter Goodman
authored
Merge pull request #55 from trailofbits/build-caching
CI Build caching
2 parents 235d1a6 + 438518e commit a513e20

File tree

6 files changed

+106
-32
lines changed

6 files changed

+106
-32
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,24 @@ jobs:
1515
ubuntu: ["20.04", "19.10", "18.04"]
1616
steps:
1717
- uses: actions/checkout@v2
18+
- uses: actions/cache@v1
19+
name: Cache build artifact store
20+
with:
21+
path: ./cache
22+
# https://github.com/actions/cache/issues/109 "Enable always writing cache to support hermetic build systems"
23+
# https://github.com/actions/cache/issues/239#issuecomment-606950711 Investigate this workaround if cache starts filling up
24+
key: store-${{ runner.os }}-${{ matrix.ubuntu }}-${{ matrix.llvm }}-${{ github.sha }}
25+
restore-keys: |
26+
store-${{ runner.os }}-${{ matrix.ubuntu }}-${{ matrix.llvm }}-
27+
store-${{ runner.os }}-${{ matrix.ubuntu }}-
28+
store-${{ runner.os }}-
1829
- name: Build LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
1930
run: |
2031
docker build . -t docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest -f Dockerfile --build-arg UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} --build-arg LLVM_VERSION=${{ matrix.llvm }}
32+
- name: Grab Cache
33+
run: |
34+
docker build . -t docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64-build:latest -f Dockerfile --build-arg UBUNTU_BASE=ubuntu:${{ matrix.ubuntu }} --build-arg LLVM_VERSION=${{ matrix.llvm }} --target cxx-common-build
35+
docker run --rm --entrypoint /bin/bash -v $(pwd)/cache:/tmp docker.pkg.github.com/trailofbits/cxx-common/llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64-build:latest -c "cp -r ./cache/* /tmp"
2136
- name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
2237
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
2338
run: |
@@ -46,6 +61,15 @@ jobs:
4661
runs-on: macos-latest
4762
steps:
4863
- uses: actions/checkout@v2
64+
- uses: actions/cache@v1
65+
name: Cache build artifact store
66+
with:
67+
path: ./cache
68+
# https://github.com/actions/cache/issues/109 "Enable always writing cache to support hermetic build systems"
69+
# https://github.com/actions/cache/issues/239#issuecomment-606950711 Investigate this workaround if cache starts filling up
70+
key: store-${{ runner.os }}-${{ github.sha }}
71+
restore-keys: |
72+
store-${{ runner.os }}-
4973
- name: Set up Python 3.8
5074
uses: actions/setup-python@v1
5175
with:

‎Dockerfile‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ RUN apt-get update && \
1616

1717
# bootstrap image should be what's needed to get a more reproducible build
1818
# environment for cxx-common
19-
FROM base as bootstrap
19+
FROM base as cxx-common-build
2020
ARG BOOTSTRAP
2121
ARG LIBRARIES
2222
ARG LLVM_VERSION
@@ -31,19 +31,24 @@ RUN pip3 install requests
3131

3232
RUN mkdir -p /cxx-common
3333
WORKDIR /cxx-common
34+
35+
# Will try to use cache at './cache'
36+
# Get cache using
37+
# docker build -t cxx-common-build --target cxx-common-build .
38+
# docker run --rm --entrypoint /bin/bash -v $(pwd)/cache:/tmp cxx-common-build -c "cp -r ./cache /tmp"
3439
COPY . ./
3540

3641
RUN mkdir -p "${BOOTSTRAP}" && mkdir -p "${LIBRARIES}"
3742

3843
RUN ./pkgman.py \
3944
--c_compiler=/usr/bin/clang \
4045
--cxx_compiler=/usr/bin/clang++ \
46+
--verbose \
47+
--use_ccache \
4148
--repository_path="${BOOTSTRAP}" \
42-
--packages=cmake && \
43-
rm -rf build && mkdir build && \
44-
rm -rf sources && mkdir sources
49+
--packages=cmake
4550

46-
RUN mkdir -p /cache && ./pkgman.py \
51+
RUN ./pkgman.py \
4752
--c_compiler=/usr/bin/clang \
4853
--cxx_compiler=/usr/bin/clang++ \
4954
--llvm_version=${LLVM_VERSION} \
@@ -52,28 +57,16 @@ RUN mkdir -p /cache && ./pkgman.py \
5257
--exclude_libcxx \
5358
"--additional_paths=${BOOTSTRAP}/cmake/bin" \
5459
"--repository_path=${LIBRARIES}" \
55-
"--packages=z3,llvm" && \
56-
rm -rf build && mkdir build && \
57-
rm -rf sources && mkdir sources && rm -rf /cache
58-
59-
# cxx-common-build should be image that contains all dependencies necessary to
60-
# build cxx-common
61-
FROM bootstrap as cxx-common-build
62-
63-
WORKDIR /cxx-common
64-
ARG BOOTSTRAP
65-
ARG LIBRARIES
60+
"--packages=z3,llvm"
6661

6762
RUN mkdir -p /cache && ./pkgman.py \
6863
--cxx_compiler="${LIBRARIES}/llvm/bin/clang++" \
6964
--c_compiler="${LIBRARIES}/llvm/bin/clang" \
70-
--use_ccache \
7165
--verbose \
66+
--use_ccache \
7267
"--additional_paths=${BOOTSTRAP}/cmake/bin:${LIBRARIES}/llvm/bin" \
7368
"--repository_path=${LIBRARIES}" \
74-
"--packages=cmake,google,xed" && \
75-
rm -rf build && mkdir build && \
76-
rm -rf sources && mkdir sources && rm -rf /cache
69+
"--packages=cmake,google,xed"
7770

7871
# dist image should be minimal artifact image
7972
FROM base as dist

‎pkgman/installers/common.py‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ def google_installer_glog(properties):
100100
print(" x Failed to create the build folder")
101101
return False
102102

103+
if properties["ccache"]:
104+
set_ccache_compiler()
105+
103106
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
104107
cmake_command += ["-DCMAKE_CXX_STANDARD=11",
105108
"-DBUILD_TESTING=OFF",
@@ -141,6 +144,9 @@ def common_installer_capstone(properties):
141144
print(" x Failed to create the build folder")
142145
return False
143146

147+
if properties["ccache"]:
148+
set_ccache_compiler()
149+
144150
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
145151
cmake_command += ["-DCMAKE_EXE_LINKER_FLAGS=-g",
146152
"-DCMAKE_C_FLAGS=-g",
@@ -247,6 +253,8 @@ def google_installer_gflags(properties):
247253
print(" x Failed to create the build folder")
248254
return False
249255

256+
if properties["ccache"]:
257+
set_ccache_compiler()
250258

251259
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
252260
cmake_command += ["-DCMAKE_INSTALL_PREFIX=" + os.path.join(repository_path, "gflags"),
@@ -289,6 +297,9 @@ def google_installer_googletest(properties):
289297
print(" x Failed to create the build folder")
290298
return False
291299

300+
if properties["ccache"]:
301+
set_ccache_compiler()
302+
292303
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator(False)
293304
cmake_command += ["-DCMAKE_CXX_STANDARD=11",
294305
"-DCMAKE_POSITION_INDEPENDENT_CODE=ON",
@@ -363,6 +374,9 @@ def google_installer_protobuf(properties):
363374
print(" x Failed to create the build folder")
364375
return False
365376

377+
if properties["ccache"]:
378+
set_ccache_compiler()
379+
366380
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator(False)
367381
cmake_command += ["-DPROTOBUF_ROOT=" + source_folder,
368382
"-DBUILD_SHARED_LIBS=OFF",
@@ -433,6 +447,9 @@ def common_installer_capnproto(properties):
433447
print(" x Failed to create the build folder")
434448
return False
435449

450+
if properties["ccache"]:
451+
set_ccache_compiler()
452+
436453
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
437454
cmake_command += ["-DCMAKE_CXX_STANDARD=11",
438455
"-DCMAKE_CXX_EXTENSIONS=ON",
@@ -590,6 +607,13 @@ def common_installer_llvm(properties):
590607
arch_list += ";AArch64;Sparc;NVPTX;ARM"
591608
arch_list += "'"
592609

610+
if properties["ccache"]:
611+
# Remove this so we don't clash with LLVM's built-in ccache config
612+
if "CMAKE_CXX_COMPILER_LAUNCHER" in os.environ:
613+
del(os.environ["CMAKE_CXX_COMPILER_LAUNCHER"])
614+
if "CMAKE_C_COMPILER_LAUNCHER" in os.environ:
615+
del(os.environ["CMAKE_C_COMPILER_LAUNCHER"])
616+
593617
cppstd = "11"
594618
if int(properties["llvm_version"]) > 900:
595619
cppstd = "14"
@@ -604,13 +628,14 @@ def common_installer_llvm(properties):
604628
cmake_command += ["-DLLVM_ENABLE_Z3_SOLVER=OFF", "-DCLANG_ANALYZER_ENABLE_Z3_SOLVER=OFF"]
605629

606630
if properties["ccache"]:
607-
print(" i Enabling ccache on /cache ... ")
631+
ccache_dir = f"{os.getcwd()}/cache/ccache"
632+
print(f" i Enabling ccache on {ccache_dir} ... ")
608633
# some versions of LLVM use CCACHE_MAX_SIZE, others use CCACHE_SIZE
609634
cmake_command.extend(
610635
["-DLLVM_CCACHE_BUILD=ON",
611-
"-DLLVM_CCACHE_SIZE=10G",
612-
"-DLLVM_CCACHE_DIR=/cache",
613-
"-DLLVM_CCACHE_MAXSIZE=10G"])
636+
"-DLLVM_CCACHE_SIZE=5G",
637+
f'-DLLVM_CCACHE_DIR="{ccache_dir}"',
638+
"-DLLVM_CCACHE_MAXSIZE=5G"])
614639

615640
if use_libcxx:
616641
if int(properties["llvm_version"]) < 371:
@@ -662,6 +687,9 @@ def common_installer_z3(properties):
662687
print(" x Failed to create the build folder")
663688
return False
664689

690+
if properties["ccache"]:
691+
set_ccache_compiler()
692+
665693
cmake_command = ["cmake"] + get_env_compiler_settings() + get_cmake_build_type(debug) + get_cmake_generator()
666694
cmake_command += ["-DZ3_BUILD_LIBZ3_SHARED=False",
667695
"-DZ3_ENABLE_EXAMPLE_TARGETS=False",

‎pkgman/installers/unix.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@ def unix_installer_boost(properties, default_toolset):
5353
if "CMAKE_CXX_COMPILER" in os.environ:
5454
os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"]
5555

56+
if properties["ccache"]:
57+
set_ccache_compiler()
58+
os.environ["CXX"] = f"ccache {os.environ['CXX']}"
59+
os.environ["CC"] = f"ccache {os.environ['CC']}"
60+
5661
configure_command = [source_folder + "/bootstrap.sh", "--prefix=" + os.path.join(repository_path, "boost"), "--with-toolset=" + toolset_name]
5762
if not run_program("Running the bootstrap script...", configure_command, source_folder, verbose=verbose_output):
5863
return False
@@ -118,7 +123,12 @@ def unix_installer_cmake(properties):
118123
if os.environ.get("CMAKE_CXX_COMPILER") is not None:
119124
os.environ["CXX"] = os.environ["CMAKE_CXX_COMPILER"]
120125

121-
if not run_program("Running the bootstrap script...", ["./bootstrap", "--verbose", "--parallel=" + str(multiprocessing.cpu_count()), "--prefix=" + destination_path], source_folder, verbose=verbose_output):
126+
enable_ccache = None
127+
if properties["ccache"]:
128+
set_ccache_compiler()
129+
enable_ccache = "--enable-ccache"
130+
131+
if not run_program("Running the bootstrap script...", ["./bootstrap", "--verbose", "--parallel=" + str(multiprocessing.cpu_count()), "--prefix=" + destination_path] + ([enable_ccache] if enable_ccache else []), source_folder, verbose=verbose_output):
122132
return False
123133

124134
if not run_program("Building the source code...", ["make", "-j" + str(multiprocessing.cpu_count())], source_folder, verbose=verbose_output):

‎pkgman/installers/utils.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,27 @@ def get_env_compiler_settings():
3232
if os.environ.get("CMAKE_C_COMPILER") is not None:
3333
cmake_compiler_settings.append("-DCMAKE_C_COMPILER=" + os.environ["CMAKE_C_COMPILER"])
3434

35+
if os.environ.get("CMAKE_CXX_COMPILER_LAUNCHER") is not None:
36+
cmake_compiler_settings.append("-DCMAKE_CXX_COMPILER_LAUNCHER=" + os.environ["CMAKE_CXX_COMPILER_LAUNCHER"])
37+
38+
if os.environ.get("CMAKE_C_COMPILER_LAUNCHER") is not None:
39+
cmake_compiler_settings.append("-DCMAKE_C_COMPILER_LAUNCHER=" + os.environ["CMAKE_C_COMPILER_LAUNCHER"])
40+
3541
return cmake_compiler_settings
3642

43+
44+
def set_ccache_compiler():
45+
"""
46+
Set the compiler environment variables to use ccache.
47+
48+
NOTE: LLVM uses its own ccache settings. Make sure these are synchronized
49+
"""
50+
os.environ["CMAKE_CXX_COMPILER_LAUNCHER"] = "ccache"
51+
os.environ["CMAKE_C_COMPILER_LAUNCHER"] = "ccache"
52+
# Only works if we never change directories in the script(s)
53+
os.environ["CCACHE_DIR"] = f'"{os.getcwd()}/cache/ccache"'
54+
55+
3756
def get_parallel_build_options():
3857
processor_count = str(multiprocessing.cpu_count())
3958

‎travis.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ osx_initialize() {
8989
xcode-select --install 2>&1 > /dev/null
9090

9191
printf " > Installing the required packages...\n"
92-
brew install coreutils
92+
brew install coreutils ccache
9393
if [ $? -ne 0 ] ; then
9494
printf " x Could not install the required dependencies\n"
9595
return 1
@@ -126,7 +126,7 @@ linux_build() {
126126
printf " > Launching the build script for CMake...\n"
127127

128128
printf "\n===\n"
129-
python3 pkgman.py --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake"
129+
python3 pkgman.py --use_ccache --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake"
130130
local pkgman_error=$?
131131
printf "===\n\n"
132132

@@ -138,7 +138,7 @@ linux_build() {
138138
printf " > Launching the build script for LLVM...\n"
139139

140140
printf "\n===\n"
141-
python3 pkgman.py --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --exclude_libcxx --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${bootstrap_repository}" "--packages=llvm"
141+
python3 pkgman.py --use_ccache --c_compiler=$(which clang) --cxx_compiler=$(which clang++) --exclude_libcxx --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${bootstrap_repository}" "--packages=llvm"
142142
local pkgman_error=$?
143143
printf "===\n\n"
144144

@@ -181,7 +181,7 @@ linux_build() {
181181
printf " > Re-launching the build script using the newly built clang...\n"
182182

183183
printf "\n===\n"
184-
python3 pkgman.py "--cxx_compiler=${bootstrap_repository}/llvm/bin/clang++" "--c_compiler=${bootstrap_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${bootstrap_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
184+
python3 pkgman.py --use_ccache "--cxx_compiler=${bootstrap_repository}/llvm/bin/clang++" "--c_compiler=${bootstrap_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${bootstrap_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
185185
local pkgman_error=$?
186186
printf "===\n\n"
187187

@@ -220,7 +220,7 @@ osx_build() {
220220
printf " > Launching the build script for CMake...\n"
221221

222222
printf "\n===\n"
223-
python3 pkgman.py --verbose "--repository_path=${bootstrap_repository}" "--packages=cmake"
223+
python3 pkgman.py --verbose --use_ccache "--repository_path=${bootstrap_repository}" "--packages=cmake"
224224
local pkgman_error=$?
225225
printf "===\n\n"
226226

@@ -232,7 +232,7 @@ osx_build() {
232232
printf " > Launching the build script for LLVM...\n"
233233

234234
printf "\n===\n"
235-
python3 pkgman.py --verbose "--additional_paths=${bootstrap_repository}/cmake/bin" "--repository_path=${library_repository}" "--packages=llvm"
235+
python3 pkgman.py --verbose --use_ccache "--additional_paths=${bootstrap_repository}/cmake/bin" "--use_ccache" "--repository_path=${library_repository}" "--packages=llvm"
236236
local pkgman_error=$?
237237
printf "===\n\n"
238238

@@ -275,7 +275,7 @@ osx_build() {
275275
printf " > Re-launching the build script using the newly built clang...\n"
276276

277277
printf "\n===\n"
278-
python3 pkgman.py "--cxx_compiler=${library_repository}/llvm/bin/clang++" "--c_compiler=${library_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${library_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
278+
python3 pkgman.py --use_ccache "--cxx_compiler=${library_repository}/llvm/bin/clang++" "--c_compiler=${library_repository}/llvm/bin/clang" --verbose "--additional_paths=${bootstrap_repository}/cmake/bin:${library_repository}/llvm/bin:${custom_bin_path}" "--repository_path=${library_repository}" "--packages=cmake,capstone,google,xed,capnproto"
279279
local pkgman_error=$?
280280
printf "===\n\n"
281281

0 commit comments

Comments
 (0)