build-all: Add support for PGO builds

Allow both doing one single stage at a time, or driving the
full build with three stages all in one command.

Author: mstorsjo

build-all.sh

@@ -76,18 +76,55 @@ while [ $# -gt 0 ]; do
     --llvm-only)
         LLVM_ONLY=1
         ;;
+    --stage1)
+        STAGE1=1
+        LLVM_ARGS="$LLVM_ARGS --disable-lldb --disable-clang-tools-extra"
+        NO_LLDB=1
+        ;;
+    --profile|--profile=*)
+        case "$1" in
+        --profile=*)
+            INSTRUMENTATION="=${1#*=}"
+            ;;
+        esac
+        PROFILE=1
+        LLVM_ARGS="$LLVM_ARGS --disable-lldb --disable-clang-tools-extra --with-clang --disable-dylib --instrumented$INSTRUMENTATION"
+        NO_LLDB=1
+        LLVM_ONLY=1
+        ;;
+    --pgo|--pgo=*)
+        PGO=1
+        LLVM_ARGS="$LLVM_ARGS --with-clang $1"
+        ;;
+    --full-pgo|--full-pgo=*)
+        case "$1" in
+        --full-pgo=*)
+            INSTRUMENTATION="=${1#*=}"
+            ;;
+        esac
+        PGO=1
+        FULL_PGO=1
+        ;;
     *)
         if [ -n "$PREFIX" ]; then
-            echo Unrecognized parameter $1
-            exit 1
+            if [ -n "$PREFIX_PGO" ]; then
+                echo Unrecognized parameter $1
+                exit 1
+            fi
+            PREFIX_PGO="$1"
+        else
+            PREFIX="$1"
         fi
-        PREFIX="$1"
         ;;
     esac
     shift
 done
 if [ -z "$PREFIX" ]; then
-    echo "$0 [--host-clang[=clang]] [--enable-asserts] [--disable-dylib] [--with-clang] [--thinlto] [--full-llvm] [--disable-lldb] [--disable-lldb-mi] [--disable-clang-tools-extra] [--host=triple] [--with-default-win32-winnt=0x601] [--with-default-msvcrt=ucrt] [--enable-cfguard|--disable-cfguard] [--no-runtimes] [--llvm-only] [--no-tools] [--wipe-runtimes] [--clean-runtimes] dest"
+    echo "$0 [--host-clang[=clang]] [--enable-asserts] [--disable-dylib] [--with-clang] [--thinlto] [--full-llvm] [--disable-lldb] [--disable-lldb-mi] [--disable-clang-tools-extra] [--host=triple] [--with-default-win32-winnt=0x601] [--with-default-msvcrt=ucrt] [--enable-cfguard|--disable-cfguard] [--no-runtimes] [--llvm-only] [--no-tools] [--wipe-runtimes] [--clean-runtimes] [--stage1] [--profile[=type]] [--pgo[=profile]] [--full-pgo[=type]] dest [pgo-dest]"
+    exit 1
+fi
+if [ -n "$PREFIX_PGO" ] && [ -z "$PGO" ] && [ -z "$FULL_PGO" ]; then
+    echo Unrecognized parameter $1
     exit 1
 fi
 
@@ -102,15 +139,80 @@ if [ -n "${HOST_CLANG}" ] && [ "${CFGUARD_ARGS}" = "--enable-cfguard"  ]; then
     "${HOST_CLANG}" -c -x c -o - - -Werror -mguard=cf </dev/null >/dev/null 2>/dev/null || CFGUARD_ARGS="--disable-cfguard"
 fi
 
+if [ -n "$FULL_PGO" ]; then
+    if [ -z "$PREFIX_PGO" ]; then
+        echo Must provide a second destination for a PGO build
+        exit 1
+    fi
+    ./build-all.sh "$PREFIX" --stage1 $LLVM_ARGS $MINGW_ARGS $CFGUARD_ARGS
+    unset COMPILER_LAUNCHER
+    ./build-all.sh "$PREFIX" --profile$INSTRUMENTATION $LLVM_ARGS
+    ./build-all.sh "$PREFIX" "$PREFIX_PGO" --thinlto --pgo --llvm-only $LLVM_ARGS
+    # If one already has a usable profile, one could also do the following
+    # two steps only:
+    # ./build-all.sh "$PREFIX" --stage1 --llvm-only
+    # ./build-all.sh "$PREFIX" "$PREFIX_PGO" --pgo
+    exit 0
+fi
+
+if [ -n "$PROFILE" ]; then
+    export PATH=$PREFIX/bin:$PATH
+    STAGE1_PREFIX=$PREFIX
+    PREFIX=/tmp/dummy-prefix
+elif [ -n "$PGO" ]; then
+    if [ -z "$PREFIX_PGO" ]; then
+        echo Must provide a second destination for a PGO build
+        exit 1
+    fi
+    export PATH=$PREFIX/bin:$PATH
+    STAGE1_PREFIX=$PREFIX
+    PREFIX=$PREFIX_PGO
+
+    if [ -n "$LLVM_ONLY" ] && [ "$PREFIX" != "$STAGE1_PREFIX" ] ; then
+        # Only rebuilding LLVM, not any runtimes. Copy the stage1 toolchain
+        # and rebuild LLVM on top of it.
+        rm -rf $PREFIX
+        mkdir -p "$(dirname "$PREFIX")"
+        cp -a "$STAGE1_PREFIX" "$PREFIX"
+        # Remove the native Linux/macOS runtimes which aren't needed in
+        # the final distribution.
+        rm -rf "$PREFIX"/lib/clang/*/lib/darwin
+        rm -rf "$PREFIX"/lib/clang/*/lib/linux
+    fi
+fi
+
+if [ "$(uname)" = "Darwin" ]; then
+    if [ -n "$PROFILE" ] || [ -n "$PGO" ]; then
+        # Using a custom Clang, which doesn't find the SDK automatically.
+        # CMake sets this automatically, but if using the compiler directly,
+        # this is needed. (If compilation uses "cc" or "gcc", it will miss
+        # the stage1 llvm-mingw toolchain and use the system compiler anyway.)
+        export SDKROOT=$(xcrun --show-sdk-path)
+    fi
+fi
+
 if [ -z "$NO_TOOLS" ]; then
     if [ -z "${HOST_CLANG}" ]; then
         ./build-llvm.sh $PREFIX $LLVM_ARGS $HOST_ARGS
+        if [ -n "$PROFILE" ]; then
+            ./pgo-training.sh llvm-project/llvm/build-instrumented $STAGE1_PREFIX
+            exit 0
+        fi
         if [ -z "$NO_LLDB" ] && [ -z "$NO_LLDB_MI" ]; then
             ./build-lldb-mi.sh $PREFIX $HOST_ARGS
         fi
         if [ -z "$FULL_LLVM" ]; then
             ./strip-llvm.sh $PREFIX $HOST_ARGS
         fi
+        if [ -n "$STAGE1" ]; then
+            if [ "$(uname)" = "Darwin" ]; then
+                ./build-llvm.sh $PREFIX --macos-native-tools
+            fi
+            # Build runtimes. On Linux, this is needed for profiling.
+            # On macOS, it is also needed for OS availability helpers like
+            # __isPlatformVersionAtLeast.
+            ./build-compiler-rt.sh --native $PREFIX
+        fi
     fi
     if [ -n "$LLVM_ONLY" ]; then
         exit 0