Build host toolchain w/o copying it from snapshots

Author: MaxDesiatov

utils/build-presets.ini

@@ -2437,39 +2437,44 @@ no-assertions
 
 [preset: webassembly]
 
-wasm
 release
-verbose
 cmake-c-launcher=%(C_CXX_LAUNCHER)s
 cmake-cxx-launcher=%(C_CXX_LAUNCHER)s
 skip-build-benchmarks
+llvm-targets-to-build=X86;WebAssembly
+install-destdir=%(INSTALL_DESTDIR)s
+swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
+llvm-install-components=clang
+install-swift
+install-prefix=/%(TOOLCHAIN_NAME)s/usr
+
+[preset: webassembly-host]
+
+mixin-preset=webassembly
+llbuild
+swiftpm
+install-swiftpm
+extra-cmake-options=
+    -DSWIFT_BUILD_SOURCEKIT=FALSE
+    -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE
+    -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
+
+[preset: webassembly-target]
+
+mixin-preset=webassembly
+wasm
+verbose
 build-stdlib-deployment-targets=wasi-wasm32
 build-swift-dynamic-sdk-overlay=false
 build-swift-dynamic-stdlib=false
 build-swift-static-sdk-overlay
 build-swift-static-stdlib
-llvm-targets-to-build=X86;WebAssembly
 stdlib-deployment-targets=wasi-wasm32
 wasi-sdk=%(SOURCE_PATH)s/wasi-sdk
 
-[preset: webassembly-installable]
-
-install_destdir=%(SOURCE_PATH)s/install
-installable_package=%(INSTALLABLE_PACKAGE)s
-install-prefix=/%(TOOLCHAIN_NAME)s/usr
-swift-install-components=autolink-driver;compiler;clang-builtin-headers;stdlib;sdk-overlay;parser-lib;editor-integration;tools;testsuite-tools;toolchain-tools;license;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers;clang-resource-dir-symlink
-llvm-install-components=clang
-install-swift
-darwin-toolchain-bundle-identifier=%(BUNDLE_IDENTIFIER)s
-darwin-toolchain-display-name=%(DISPLAY_NAME)s
-darwin-toolchain-display-name-short=%(DISPLAY_NAME_SHORT)s
-darwin-toolchain-name=%(TOOLCHAIN_NAME)s
-darwin-toolchain-version=%(TOOLCHAIN_VERSION)s
-darwin-toolchain-alias=swift
-
 [preset: webassembly-linux]
 
-mixin-preset=webassembly
+mixin-preset=webassembly-target
 extra-cmake-options=
     -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
     -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2484,7 +2489,7 @@ extra-cmake-options=
 
 [preset: webassembly-macos]
 
-mixin-preset=webassembly
+mixin-preset=webassembly-target
 extra-cmake-options=
     -DWASI_ICU_URL:STRING="https://github.com/swiftwasm/icu4c-wasi/releases/download/0.5.0/icu4c-wasi.tar.xz"
     -DWASI_ICU_MD5:STRING="25943864ebbfff15cf5aee8d9d5cc4d7"
@@ -2496,15 +2501,3 @@ extra-cmake-options=
     -DSWIFT_BUILD_SYNTAXPARSERLIB=FALSE
     -DCMAKE_AR='/usr/local/opt/llvm/bin/llvm-ar'
     -DCMAKE_RANLIB='/usr/local/opt/llvm/bin/llvm-ranlib'
-
-[preset: webassembly-linux-installable]
-
-mixin-preset=
-    webassembly-linux
-    webassembly-installable
-
-[preset: webassembly-macos-installable]
-
-mixin-preset=
-    webassembly-macos
-    webassembly-installable

utils/webassembly/build-swiftpm.sh

@@ -9,11 +9,11 @@ WASI_SDK_PATH=$SOURCE_PATH/wasi-sdk
 case $(uname -s) in
   Darwin)
     OS_SUFFIX=osx
-    PRESET_NAME=webassembly-macos-installable
+    PRESET_NAME=webassembly-macos
   ;;
   Linux)
     OS_SUFFIX=linux
-    PRESET_NAME=webassembly-linux-installable
+    PRESET_NAME=webassembly-linux
   ;;
   *)
     echo "Unrecognised platform $(uname -s)"
@@ -35,47 +35,50 @@ BUNDLE_IDENTIFIER="swiftwasm.${YEAR}${MONTH}${DAY}"
 DISPLAY_NAME_SHORT="Swift for WebAssembly Development Snapshot"
 DISPLAY_NAME="${DISPLAY_NAME_SHORT} ${YEAR}-${MONTH}-${DAY}"
 
-$SOURCE_PATH/swift/utils/build-script --preset=$PRESET_NAME \
+HOST_TOOLCHAIN_DESTDIR=$SOURCE_PATH/host-toolchain-sdk
+HOST_TOOLCHAIN_SDK=$HOST_TOOLCHAIN_DESTDIR/$TOOLCHAIN_NAME
+
+# Build the host toolchain and SDK first.
+$SOURCE_PATH/swift/utils/build-script --preset=webassembly-host \
+  INSTALL_DESTDIR="$HOST_TOOLCHAIN_DESTDIR" \
+  TOOLCHAIN_NAME="$TOOLCHAIN_NAME" \
+  C_CXX_LAUNCHER="$(which sccache)"
+
+# Clean up the host toolchain build directory so that the next
+# `build-script` invocation doesn't pick up wrong CMake config files.
+# For some reason passing `--reconfigure` to `build-script` won't do this.
+rm -rf $SOURCE_PATH/build/Ninja-ReleaseAssert/swift-macosx-x86_64
+
+# build the cross-compilled toolchain
+$SOURCE_PATH/swift/utils/build-script \
+  --preset=$PRESET_NAME \
+  INSTALL_DESTDIR="$SOURCE_PATH/install" \
   SOURCE_PATH="$SOURCE_PATH" \
-  INSTALLABLE_PACKAGE="$INSTALLABLE_PACKAGE" \
   BUNDLE_IDENTIFIER="${BUNDLE_IDENTIFIER}" \
   DISPLAY_NAME="${DISPLAY_NAME}" \
   DISPLAY_NAME_SHORT="${DISPLAY_NAME_SHORT}" \
   TOOLCHAIN_NAME="${TOOLCHAIN_NAME}" \
   TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" \
   C_CXX_LAUNCHER="$(which sccache)"
 
-
-NIGHTLY_TOOLCHAIN=$SOURCE_PATH/swift-nightly-toolchain
-if [ ! -e $NIGHTLY_TOOLCHAIN ]; then
-  $UTILS_PATH/install-nightly-toolchain.sh
-fi
-
-TMP_DIR=$(mktemp -d)
-cd $TMP_DIR
-tar xfz $INSTALLABLE_PACKAGE $TOOLCHAIN_NAME
-cd $TMP_DIR/$TOOLCHAIN_NAME
-
-# Merge wasi-sdk and toolchain
-cp -r $WASI_SDK_PATH/lib/clang usr/lib
-cp -a $WASI_SDK_PATH/bin/{*ld,llvm-ar} usr/bin
-cp -r $WASI_SDK_PATH/share/wasi-sysroot usr/share
-
-# Build SwiftPM and install it into toolchain
-$UTILS_PATH/build-swiftpm.sh $TMP_DIR/$TOOLCHAIN_NAME
+# Merge wasi-sdk and the toolchain
+cp -a $WASI_SDK_PATH/lib/clang $HOST_TOOLCHAIN_SDK/usr/lib
+cp -a $WASI_SDK_PATH/bin/{clang*,*ld,llvm-ar} $HOST_TOOLCHAIN_SDK/usr/bin
+cp -r $WASI_SDK_PATH/share/wasi-sysroot $HOST_TOOLCHAIN_SDK/usr/share
 
 # Replace absolute sysroot path with relative path
-sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
+sed -i -e "s@\".*/include@\"../../../../share/wasi-sysroot/include@g" $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi/wasm32/glibc.modulemap
 
-# Copy nightly-toolchain's host environment stdlib into toolchain
+# Copy host environment stdlib into toolchain
 
 if [[ "$(uname)" == "Linux" ]]; then
-  # Avoid to copy usr/lib/swift/clang because our toolchain's one is a directory
-  # but nightly's one is symbolic link, so fail to merge them.
-  rsync -a $NIGHTLY_TOOLCHAIN/usr/lib/ $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/ --exclude 'swift/clang'
+  # Avoid copying usr/lib/swift/clang because our toolchain's one is a directory
+  # but nightly's one is symbolic link. A simple copy fails to merge them.
+  rsync -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib $HOST_TOOLCHAIN_SDK/usr/lib/ --exclude 'swift/clang'
 else
-  cp -r $NIGHTLY_TOOLCHAIN/usr/lib/swift/macosx $TMP_DIR/$TOOLCHAIN_NAME/usr/lib/swift
+  cp -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift_static $HOST_TOOLCHAIN_SDK/usr/lib/swift_static
+  cp -a $SOURCE_PATH/install/$TOOLCHAIN_NAME/usr/lib/swift/wasi $HOST_TOOLCHAIN_SDK/usr/lib/swift
 fi
 
-cd $TMP_DIR
+cd $HOST_TOOLCHAIN_DESTDIR
 tar cfz $PACKAGE_ARTIFACT $TOOLCHAIN_NAME