Skip to content

Commit be224c7

Browse files
Preserve LLVM and Clang libraries in the final stage (#14)
* Docs: Update required dependencies in the README * final stage: Preserve the LLVM and Clang libraries * Docs: Update the README to document how to create the release tarball * stage 0: Prevent the use of links for the destination folder * Code review changes * build: Add a script to build the toolchain inside Docker * build: Use Ubuntu 18.04 as base Docker image * build: Rename the docker script * Docs: Update the README
1 parent 203e5d6 commit be224c7

File tree

4 files changed

+141
-10
lines changed

4 files changed

+141
-10
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
build
2+

README.md

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ For the instructions we will use Ubuntu 18.04.
3636

3737
## Prerequisites
3838
```
39-
sudo apt install --no-install-recommends g++-8 gcc-8 automake autoconf gettext bison flex unzip help2man libtool-bin libncurses-dev make ninja-build patch
39+
sudo apt install g++-8 gcc-8 automake autoconf gettext bison flex unzip help2man libtool-bin libncurses-dev make ninja-build patch txinfo gawk wget git texinfo xz-utils python
4040
```
4141
Then use `update-alternatives` to tell the system that the version of GCC/G++ and CPP is the default we would like to use:
4242
```
@@ -65,7 +65,13 @@ The script has to be run as a normal user and accepts one argument, which is the
6565
This should output the sysroot under `/opt/osquery-toolchain/final` and the LLVM toolchain will be under `/opt/osquery-toolchain/final/sysroot/usr`
6666

6767
## Redistributing and usage
68-
tar the sysroot folder and uncompress that wherever you like on the target machine.
68+
1. Enter inside the **final** folder within the destination path
69+
2. Rename the **sysroot** folder to **osquery-toolchain**
70+
3. Compress the folder with the following command: `tar -pcvJf osquery-toolchain-<VERSION>.tar.xz osquery-toolchain`
71+
72+
Make sure that the **<VERSION>** field matches a valid tag, for example: `osquery-toolchain-1.0.1.tar.xz`.
73+
74+
The generated tarball can then be uncompressed wherever you like on the target machine.
6975

7076
The toolchain defaults to using libc++ as the C++ standard library, compiler-rt as part of the builtins it needs, instead of relying on libgcc and lld as the linker; so these are implicit.
7177
libc++abi has to be explicitly linked when compiling C++ with `l:libc++abi.a` instead; merged with it there's also libunwind, which is therefore implicit.

build.sh

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,6 @@ source ./config
229229
TOOLCHAIN_DIR=$1
230230
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
231231

232-
233232
# We are already at the final stage, nothing to do
234233
if [ -e $TOOLCHAIN_DIR/final/sysroot ]; then
235234
echo "Nothing to do. If you want to redo the final stage please delete the $TOOLCHAIN_DIR/final/sysroot folder"
@@ -321,7 +320,6 @@ additional_compiler_flags="-s" \
321320
additional_cmake="" \
322321
build_llvm
323322

324-
325323
build_folder="build-compilerrt-builtins" \
326324
cc_compiler="clang" \
327325
cxx_compiler="clang++" \
@@ -379,16 +377,12 @@ additional_linker_flags="-rtlib=compiler-rt -l:libc++abi.a -ldl -lpthread" \
379377
additional_cmake="${llvm_additional_cmake}" \
380378
build_llvm
381379

382-
383380
CURRENT_DIR=$TOOLCHAIN_DIR/final
384381
SYSROOT=$TOOLCHAIN_DIR/final/$TUPLE/$TUPLE/sysroot
385382
PREFIX=$SYSROOT/usr
386383

387-
# Remove the static libclang/liblld/libLLVM and all versions of libstdc++ from the sysroot.
384+
# Remove all the versions of libstdc++ from the sysroot.
388385
( cd $PREFIX/lib; \
389-
rm -f libclang*.a; \
390-
rm -f liblld*.a; \
391-
rm -f libLLVM*.a; \
392386
rm -f libstdc*)
393387

394388
# Remove unused GCC binaries
@@ -436,12 +430,13 @@ do
436430
done
437431

438432
remaining_symlinks=`find "$SYSROOT" -type l`
433+
real_sysroot=`realpath ${SYSROOT}`
439434

440435
symlinks_to_fix=0
441436
while read line
442437
do
443438
real_path=`readlink -f "$line"`
444-
if [[ ! "$real_path" == "$SYSROOT/"* ]]; then
439+
if [[ ! "$real_path" == "$real_sysroot/"* ]]; then
445440
symlinks_to_fix=1
446441
echo "$line -> $real_path"
447442
fi

build_inside_docker.sh

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/usr/bin/env bash
2+
3+
export CMAKE_VERSION="3.15.2"
4+
export BASE_IMAGE="ubuntu:bionic"
5+
6+
main() {
7+
if [[ -z "${run_build_script}" ]] ; then
8+
startDockerContainer || return 1
9+
else
10+
buildOsqueryToolchain || return 1
11+
fi
12+
13+
return 0
14+
}
15+
16+
startDockerContainer() {
17+
if [[ -d "build" ]] ; then
18+
echo "Reusing existing build folder"
19+
else
20+
echo "Creating new build folder"
21+
22+
mkdir build
23+
if [[ $? != 0 ]] ; then
24+
echo "Failed to create the build folder"
25+
return 1
26+
fi
27+
fi
28+
29+
local container_name="osquery-toolchain-$(git rev-parse HEAD)"
30+
docker rm "${container_name}" > /dev/null 2>&1
31+
32+
docker run --rm -e "run_build_script=1" -v "$(realpath build):/opt/osquery-toolchain" -v "$(pwd):/home/osquery/osquery-toolchain" --name "${container_name}" -it "${BASE_IMAGE}" /bin/bash -c '/home/osquery/osquery-toolchain/build_inside_docker.sh'
33+
if [[ $? != 0 ]] ; then
34+
echo "Failed to start the Docker container"
35+
return 1
36+
fi
37+
38+
return 0
39+
}
40+
41+
buildOsqueryToolchain() {
42+
# https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself
43+
local home_folder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
44+
cd "${home_folder}"
45+
46+
installSystemDependencies || return 1
47+
installCMake || return 1
48+
initializeOsqueryUser || return 1
49+
50+
export KEEP_INTERMEDIATE_STAGES=1
51+
sudo -u osquery ./build.sh "/opt/osquery-toolchain"
52+
if [[ $? != 0 ]] ; then
53+
echo "The build script has failed"
54+
return 1
55+
fi
56+
57+
return 0
58+
}
59+
60+
initializeOsqueryUser() {
61+
useradd -d "/home/osquery" -M osquery
62+
if [[ $? != 0 ]] ; then
63+
echo "Failed to create the osquery user"
64+
return 1
65+
fi
66+
67+
chown -R osquery:osquery "/home/osquery"
68+
if [[ $? != 0 ]] ; then
69+
echo "Failed to set the require permissions on the home directory"
70+
return 1
71+
fi
72+
73+
return 0
74+
}
75+
76+
installSystemDependencies() {
77+
apt-get update
78+
if [[ $? != 0 ]] ; then
79+
echo "Failed to update the package repositories"
80+
return 1
81+
fi
82+
83+
apt-get install g++-8 gcc-8 automake autoconf gettext bison flex unzip help2man libtool-bin libncurses-dev make ninja-build wget git texinfo xz-utils gawk python sudo -y
84+
if [[ $? != 0 ]] ; then
85+
echo "Failed to install the required dependencies"
86+
return 1
87+
fi
88+
89+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 20
90+
if [ $? != 0 ] ; then
91+
echo "Failed to set the default gcc binary path"
92+
return 1
93+
fi
94+
95+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 20
96+
if [ $? != 0 ] ; then
97+
echo "Failed to set the default g++ binary path"
98+
return 1
99+
fi
100+
101+
update-alternatives --install /usr/bin/cpp cpp /usr/bin/cpp-8 20
102+
if [ $? != 0 ] ; then
103+
echo "Failed to set the default cpp binary path"
104+
return 1
105+
fi
106+
107+
return 0
108+
}
109+
110+
installCMake() {
111+
local cmake_archive_path="/tmp/cmake.tar.gz"
112+
wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" -O "${cmake_archive_path}"
113+
if [[ $? != 0 ]] ; then
114+
echo "Failed to download the CMake release archive"
115+
return 1
116+
fi
117+
118+
tar xvf "${cmake_archive_path}" -C "/usr/local" --strip 1
119+
if [[ $? != 0 ]] ; then
120+
echo "Failed to extract the CMake release archive"
121+
return 1
122+
fi
123+
124+
return 0
125+
}
126+
127+
main $@
128+
exit $?

0 commit comments

Comments
 (0)