Add release-6.1 scheme

.github/scripts/build-matrix.rb

@@ -8,7 +8,6 @@
     "target": "ubuntu18.04_x86_64",
     "containers": {
       # "main": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-18.04",
-      # "release-5.9": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-18.04@sha256:d5d555cf48fc02f5003928a064c03b76c012fef8afbe54a1942c6bfc3d773c58",
       "release-5.10": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-18.04@sha256:d5d555cf48fc02f5003928a064c03b76c012fef8afbe54a1942c6bfc3d773c58"
     },
     "run_stdlib_test": true,
@@ -25,9 +24,9 @@
     "target": "ubuntu20.04_x86_64",
     "containers": {
       "main": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04",
-      # "release-5.9": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:cc1b99e352ee207da2c75c7bcf81aa8b1d2c08215fd1d05dc0777c40a62f31f1",
       "release-5.10": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:cc1b99e352ee207da2c75c7bcf81aa8b1d2c08215fd1d05dc0777c40a62f31f1",
       "release-6.0": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:9007661fb6d57ebef902618e831458e88068350b0ecc8d85e56441d25a9f4312",
+      "release-6.1": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:9007661fb6d57ebef902618e831458e88068350b0ecc8d85e56441d25a9f4312",
     },
     "run_stdlib_test": true,
     "run_full_test": false,
@@ -43,9 +42,9 @@
     "target": "ubuntu22.04_x86_64",
     "containers": {
       "main": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-22.04",
-      # "release-5.9": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:adfa0a8fbc6e5cc7ce5e38a5a9406d4fa5c557871204a65f0690478022d6b359",
       "release-5.10": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:adfa0a8fbc6e5cc7ce5e38a5a9406d4fa5c557871204a65f0690478022d6b359",
       "release-6.0": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:8060adba9c044b1fc751b9671167320c9cc0094d0a45d7b8fedb707c49b764c2",
+      "release-6.1": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:8060adba9c044b1fc751b9671167320c9cc0094d0a45d7b8fedb707c49b764c2",
     },
     "run_stdlib_test": true,
     "run_full_test": false,
@@ -61,9 +60,9 @@
     "target": "amazonlinux2_x86_64",
     "containers": {
       "main": "ghcr.io/swiftwasm/swift-ci:main-amazon-linux-2",
-      # "release-5.9": "ghcr.io/swiftwasm/swift-ci:main-amazon-linux-2@sha256:d5264ac43e935249b1c8777f6809ebbd2836cb0e8f7dac3bfeeb0b3cdb479b70",
       "release-5.10": "ghcr.io/swiftwasm/swift-ci:main-amazon-linux-2@sha256:d5264ac43e935249b1c8777f6809ebbd2836cb0e8f7dac3bfeeb0b3cdb479b70",
       "release-6.0": "ghcr.io/swiftwasm/swift-ci:main-amazon-linux-2@sha256:fc95912c595faecacbb869635fa871f8f31b8d6ed2f0df6792b53628db7ada94",
+      "release-6.1": "ghcr.io/swiftwasm/swift-ci:main-amazon-linux-2@sha256:fc95912c595faecacbb869635fa871f8f31b8d6ed2f0df6792b53628db7ada94",
     },
     "run_stdlib_test": false,
     "run_full_test": false,
@@ -87,7 +86,7 @@
   {
     "build_os": "macos-14",
     "agent_query": "macos-14",
-    "schemes": ["release-5.10", "release-6.0", "main"],
+    "schemes": ["release-5.10", "release-6.0", "release-6.1", "main"],
     "target": "macos_arm64",
     "run_stdlib_test": true,
     "run_full_test": false,
@@ -97,18 +96,6 @@
     "free_disk_space": true,
     "only_swift_sdk": false,
   },
-  {
-    "build_os": "macos-13",
-    "agent_query": ["self-hosted", "macOS", "ARM64"],
-    "schemes": ["release-5.9"],
-    "target": "macos_arm64",
-    "run_stdlib_test": true,
-    "run_full_test": false,
-    "run_e2e_test": true,
-    "build_hello_wasm": true,
-    "clean_build_dir": true,
-    "only_swift_sdk": false,
-  },
   # Generic Swift SDK build
   {
     "job_name": "Swift SDK",
@@ -118,6 +105,7 @@
     "containers": {
       "main": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-22.04",
       "release-6.0": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:8060adba9c044b1fc751b9671167320c9cc0094d0a45d7b8fedb707c49b764c2",
+      "release-6.1": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:8060adba9c044b1fc751b9671167320c9cc0094d0a45d7b8fedb707c49b764c2",
     },
     "run_stdlib_test": true,
     "run_full_test": false,
@@ -186,9 +174,9 @@ def main
         "target": "ubuntu20.04_aarch64",
         "containers": {
           "main": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04",
-          # "release-5.9": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:0e04dd550557d9f4f773bda55a6ac355c7c9696ea6efc3e59318bd49569aa00e",
           "release-5.10": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:0e04dd550557d9f4f773bda55a6ac355c7c9696ea6efc3e59318bd49569aa00e",
           "release-6.0": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:2a977060476a0f6f702d59ed0cd4e8506c7c1f1ed48d85a1a684d2d3fb62c297",
+          "release-6.1": "ghcr.io/swiftwasm/swift-ci:main-ubuntu-20.04@sha256:2a977060476a0f6f702d59ed0cd4e8506c7c1f1ed48d85a1a684d2d3fb62c297",
         },
         "run_stdlib_test": false,
         "run_full_test": false,
@@ -202,7 +190,7 @@ def main
   end
 
   schemes = derive_schemes(options)
-  schemes = schemes - ["release-5.9"] # Skip 5.9 for now
+  schemes = schemes
 
   matrix_entries = schemes.flat_map do |scheme|
     if scheme == "main"

docs/release.md

@@ -25,22 +25,31 @@ We are usually doing the following steps:
 1. Create a new release scheme in [swiftwasm-build's `./schemes` directory](https://github.com/swiftwasm/swiftwasm-build/tree/main/schemes) by copying the patches from `main` scheme at the moment.
     ```console
     $ cp -r schemes/main schemes/release-5.x
-    $ vim schemes/release-5.x/manifest.json # Update the fields for the new release
     ```
-2. Update CI configuration to build the new release scheme:
+2. Update `schemes/<release-scheme>/manifest.json`
+    - `base-tag` field should be updated to the tag of the new release branch.
+    - `update-checkout-scheme` field should be updated to the one passed to `--scheme` option of [`utils/update-checkout`](https://github.com/swiftlang/swift/blob/main/utils/update-checkout) script.
+    - `swift-org-download-channel` field should be updated to the new release branch.
+        e.g. If a download link of a new release snapshot is "https://download.swift.org/swift-6.1-branch/xcode/swift-6.1-DEVELOPMENT-SNAPSHOT-2024-11-19-a/swift-6.1-DEVELOPMENT-SNAPSHOT-2024-11-19-a-osx.pkg", the value should be "swift-6.1-branch".
+3. Update `.github/scripts/build-matrix.rb` to pin the builder container image version for the new release scheme:
+3. Update CI configuration to build the new release scheme:
     - [`.github/workflows/nightly-distribution.yml`](https://github.com/swiftwasm/swift/blob/0895044e2ba31ccd1aade8068088b1fd3137fffb/.github/workflows/nightly-distribution.yml#L8-L11)
 
 Once the upstream publishes their official release, we are doing the following steps:
 
-1. Quality assurance of our latest release candidate snapshot toolchain
+1. Update `schemes/<release-scheme>/manifest.json`
+    - `base-tag` field should be updated to the tag of the official release.
+    - `swift-org-download-channel` field should be updated to the new release branch.
+        e.g. If a download link of a new release is `https://download.swift.org/swift-6.0.2-release/xcode/swift-6.0.2-RELEASE/swift-6.0.2-RELEASE-osx.pkg`, the value should be "swift-6.0.2-release".
+2. Quality assurance of our latest release candidate snapshot toolchain
     1. Check core libraries and tools we are maintaining work properly with the toolchain. List of the libraries and tools we are maintaining:
         - [JavaScriptKit](https://github.com/swiftwasm/JavaScriptKit): See [the past PR](https://github.com/swiftwasm/JavaScriptKit/pull/227)
         - [carton](https://github.com/swiftwasm/carton): See [the past PR](https://github.com/swiftwasm/carton/pull/398)
         - [switwasm-docker](https://github.com/swiftwasm/swiftwasm-docker): Add a new Docker image for the new release if it's major release. Update the existing Docker image if it's minor release.
         - [swiftwasm-action](https://github.com/swiftwasm/swiftwasm-action): Update the base carton image version.
         - [setup-swiftwasm](https://github.com/swiftwasm/setup-swiftwasm): Update the default toolchain version.
     2. Collect feedback from the community by asking them to try a release candidate snapshot.
-2. Once we are ready to release, trigger GitHub Actions workflow by running the following command:
+3. Once we are ready to release, trigger GitHub Actions workflow by running the following command:
 
     ```console
     $ gh workflow run manual-distribution.yml --repo swiftwasm/swift -f scheme=5.9 -f run-id=<replace-run-id> -f override-name=swift-wasm-5.9.0-RELEASE -f display_name="Swift for WebAssembly 5.9.0 Release $(date +'%Y-%m-%d')" -f display_name_short="Swift for WebAssembly 5.9.0 Release"

schemes/release-6.1/build/build-foundation.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -ex
+DESTINATION_TOOLCHAIN=$1
+LLVM_BIN_DIR=$2
+CLANG_BIN_DIR=$3
+SWIFT_BIN_DIR=$4
+WASI_SYSROOT_PATH=$5
+TRIPLE="$6"
+
+SOURCE_PATH="$(cd "$(dirname $0)/../../../.." && pwd)"
+SCHEME_BUILD_PATH="$(cd "$(dirname $0)" && pwd)"
+BUILD_SDK_PATH="$SOURCE_PATH/build-sdk"
+LIBXML2_PATH="$BUILD_SDK_PATH/libxml2-$TRIPLE"
+
+FOUNDATION_BUILD="$SOURCE_PATH/build/WebAssembly/foundation-$TRIPLE"
+FOUNDATION_MACROS_BUILD="$SOURCE_PATH/build/WebAssembly/foundation-macros-$TRIPLE"
+
+swift_extra_flags=""
+c_extra_flags=""
+if [[ "$TRIPLE" == "wasm32-unknown-wasip1-threads" ]]; then
+  swift_extra_flags="-Xcc -matomics -Xcc -mbulk-memory -Xcc -mthread-model -Xcc posix -Xcc -pthread -Xcc -ftls-model=local-exec"
+  c_extra_flags="-mthread-model posix -pthread -ftls-model=local-exec"
+fi
+
+cmake -G Ninja \
+  -D CMAKE_C_COMPILER="$CLANG_BIN_DIR/clang" \
+  -D CMAKE_CXX_COMPILER="$CLANG_BIN_DIR/clang++" \
+  -D CMAKE_Swift_COMPILER="$SWIFT_BIN_DIR/swiftc" \
+  -D BUILD_SHARED_LIBS=ON \
+  -B "$FOUNDATION_MACROS_BUILD" \
+  "${SOURCE_PATH}/swift-foundation/Sources/FoundationMacros"
+
+cmake --build "$FOUNDATION_MACROS_BUILD"
+
+cmake -G Ninja \
+  -D CMAKE_BUILD_TYPE="Release" \
+  -D CMAKE_SYSROOT="$WASI_SYSROOT_PATH" \
+  -D CMAKE_Swift_COMPILER="$SWIFT_BIN_DIR/swiftc" \
+  -D CMAKE_STAGING_PREFIX="$DESTINATION_TOOLCHAIN/usr" \
+  -D CMAKE_SYSTEM_NAME=WASI \
+  -D CMAKE_SYSTEM_PROCESSOR=wasm32 \
+  -D CMAKE_C_COMPILER_TARGET="$TRIPLE" \
+  -D CMAKE_CXX_COMPILER_TARGET="$TRIPLE" \
+  -D CMAKE_Swift_COMPILER_TARGET="$TRIPLE" \
+  -D CMAKE_C_COMPILER="$CLANG_BIN_DIR/clang" \
+  -D CMAKE_CXX_COMPILER="$CLANG_BIN_DIR/clang++" \
+  -D CMAKE_AR="$LLVM_BIN_DIR/llvm-ar" \
+  -D CMAKE_RANLIB="$LLVM_BIN_DIR/llvm-ranlib" \
+  -D LIBXML2_INCLUDE_DIR="$LIBXML2_PATH/include/libxml2" \
+  -D LIBXML2_LIBRARY="$LIBXML2_PATH/lib" \
+  -D BUILD_SHARED_LIBS=OFF \
+  -D FOUNDATION_BUILD_TOOLS=OFF \
+  -D CMAKE_Swift_COMPILER_FORCED=ON \
+  -D CMAKE_C_COMPILER_FORCED=ON \
+  -D CMAKE_CXX_COMPILER_FORCED=ON \
+  -D CMAKE_Swift_FLAGS="-sdk $WASI_SYSROOT_PATH -resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static $swift_extra_flags" \
+  -D CMAKE_C_FLAGS="-resource-dir $DESTINATION_TOOLCHAIN/usr/lib/swift_static/clang -B $LLVM_BIN_DIR $c_extra_flags" \
+  -D _SwiftCollections_SourceDIR="$SOURCE_PATH/swift-collections" \
+  -D _SwiftFoundation_SourceDIR="$SOURCE_PATH/swift-foundation" \
+  -D _SwiftFoundationICU_SourceDIR="$SOURCE_PATH/swift-foundation-icu" \
+  -D SwiftFoundation_MACRO="$FOUNDATION_MACROS_BUILD/lib" \
+  -B "$FOUNDATION_BUILD" \
+  "${SOURCE_PATH}/swift-corelibs-foundation"
+
+cmake --build "$FOUNDATION_BUILD"
+cmake --install "$FOUNDATION_BUILD"

schemes/release-6.1/build/build-target-toolchain.sh

@@ -0,0 +1,180 @@
+#!/bin/bash
+#
+# Build the Swift standard library.
+
+set -euo pipefail
+set -x
+
+print_help() {
+  echo "Usage: $0 [options]"
+  echo ""
+  echo "Options:"
+  echo "  --help               Display this help message."
+  echo "  --llvm-bin           Path to LLVM bin directory."
+  echo "  --swift-bin          Path to Swift bin directory."
+}
+
+SCHEMES_BUILD_PATH="$(cd "$(dirname "$0")" && pwd)"
+SOURCE_PATH="$(cd "$(dirname "$0")/../../../.." && pwd)"
+TARGET_BUILD_ROOT=$SOURCE_PATH/build/WebAssembly
+WASI_SYSROOT_PATH="$TARGET_BUILD_ROOT/wasi-sysroot"
+PACKAGING_DIR="$SOURCE_PATH/build/Packaging"
+TARGET_TOOLCHAIN_DESTDIR=$PACKAGING_DIR/target-toolchain
+
+build_target_toolchain() {
+
+  local LLVM_BIN_DIR="$1"
+  local CLANG_BIN_DIR="$2"
+  local SWIFT_BIN_DIR="$3"
+  local TRIPLE="$4"
+  local SHORT_TRIPLE="$5"
+  local CLANG_MULTIARCH_TRIPLE="$6"
+  local STDLIB_PRODUCT="$7"
+  local COMPILER_RT_OS_DIR="$8"
+
+  local HOST_SUFFIX
+  HOST_SUFFIX=$(find "$TARGET_BUILD_ROOT" -name "wasmstdlib-*" -exec basename {} \; | sed 's/wasmstdlib-//')
+
+  local TRIPLE_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR/$TRIPLE"
+
+  env DESTDIR="$TRIPLE_DESTDIR" \
+    cmake --install "$TARGET_BUILD_ROOT/$STDLIB_PRODUCT-$HOST_SUFFIX" --prefix /usr
+
+  local swift_testing_build_dir="$TARGET_BUILD_ROOT/swift-testing-$SHORT_TRIPLE"
+  # TODO: Remove this check once we build swift-testing for +threads target
+  if [[ -d "$swift_testing_build_dir" ]]; then
+    env DESTDIR="$TRIPLE_DESTDIR" \
+      cmake --install "$swift_testing_build_dir" --prefix /usr
+  fi
+
+  rm -rf "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR"
+  # XXX: Is this the right way to install compiler-rt?
+  cp -R "$TARGET_BUILD_ROOT/wasi-sysroot/$CLANG_MULTIARCH_TRIPLE/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/swift_static/clang/lib/$COMPILER_RT_OS_DIR"
+
+  # FIXME: Clang resource directory installation is not the best way currently.
+  # We currently have two copies of compiler headers copied from the base toolchain in
+  # lib/swift/clang and lib/swift_static/clang. This is because the Swift CMake build
+  # system installs the compiler headers from the native tools path when not building
+  # tools including clang compiler. This is not ideal but then where should we bring
+  # the compiler headers from? If we use the headers beside the base toolchain, clang
+  # driver will not be able to find libclang_rt.builtins-wasm32.a because it is not
+  # a part of the base toolchain. We need to find a better way to handle this.
+  local CLANG_VERSION
+  CLANG_VERSION="$(basename "$($CLANG_BIN_DIR/clang -print-resource-dir)")"
+  mkdir -p "$TRIPLE_DESTDIR/usr/lib/clang/$CLANG_VERSION/lib"
+  ln -sf "../../../swift_static/clang/lib/$COMPILER_RT_OS_DIR" "$TRIPLE_DESTDIR/usr/lib/clang/$CLANG_VERSION/lib/$COMPILER_RT_OS_DIR"
+}
+
+build_target_corelibs() {
+  local LLVM_BIN_DIR="$1"
+  local CLANG_BIN_DIR="$2"
+  local SWIFT_BIN_DIR="$3"
+  local TRIPLE="$4"
+  local SHORT_TRIPLE="$5"
+
+  local TRIPLE_DESTDIR="$TARGET_TOOLCHAIN_DESTDIR/$TRIPLE"
+  local CORELIBS_ARGS=(
+    "$TRIPLE_DESTDIR"
+    "$LLVM_BIN_DIR"
+    "$CLANG_BIN_DIR"
+    "$SWIFT_BIN_DIR"
+    "$WASI_SYSROOT_PATH/$SHORT_TRIPLE"
+  )
+  "$SCHEMES_BUILD_PATH/build-foundation.sh" "${CORELIBS_ARGS[@]}" "$TRIPLE"
+  "$SCHEMES_BUILD_PATH/build-xctest.sh" "${CORELIBS_ARGS[@]}" "$TRIPLE"
+}
+
+main() {
+  local OPTIONS_LLVM_BIN=""
+  local OPTIONS_CLANG_BIN=""
+  local OPTIONS_SWIFT_BIN=""
+
+  while [[ $# -gt 0 ]]; do
+    case "$1" in
+      --llvm-bin)
+        OPTIONS_LLVM_BIN="$2"
+        shift 2
+        ;;
+      --clang-bin)
+        OPTIONS_CLANG_BIN="$2"
+        shift 2
+        ;;
+      --swift-bin)
+        OPTIONS_SWIFT_BIN="$2"
+        shift 2
+        ;;
+      --scheme)
+        OPTIONS_SCHEME="$2"
+        shift 2
+        ;;
+      --help)
+        print_help
+        exit 0
+        ;;
+      *)
+        echo "Unknown option: $1"
+        print_help
+        exit 1
+        ;;
+    esac
+  done
+
+  if [[ -z "$OPTIONS_LLVM_BIN" ]]; then
+    echo "Missing --llvm-bin option"
+    print_help
+    exit 1
+  fi
+
+  if [[ -z "$OPTIONS_SWIFT_BIN" ]]; then
+    echo "Missing --swift-bin option"
+    print_help
+    exit 1
+  fi
+
+  if [[ -z "$OPTIONS_CLANG_BIN" ]]; then
+    OPTIONS_CLANG_BIN="$OPTIONS_LLVM_BIN"
+  fi
+
+  # NOTE: The llvm-cmake-options is a workaround for the issue on amazonlinux2
+  # See https://github.com/apple/swift/commit/40c7268e8f7d402b27e3ad16a84180e07c37f92c
+  "$SOURCE_PATH/swift/utils/build-script" \
+    --build-subdir=WebAssembly \
+    --release \
+    --skip-build-llvm \
+    --skip-build-swift \
+    --skip-build-cmark \
+    --skip-build-benchmarks \
+    --skip-early-swift-driver \
+    --build-wasm-stdlib \
+    --skip-test-wasm-stdlib \
+    --native-swift-tools-path="$OPTIONS_SWIFT_BIN" \
+    --native-clang-tools-path="$OPTIONS_CLANG_BIN" \
+    --native-llvm-tools-path="$OPTIONS_LLVM_BIN" \
+    --extra-cmake-options="\
+      -DSWIFT_STDLIB_TRACING=NO \
+      -DSWIFT_STDLIB_HAS_ASL=NO \
+      -DSWIFT_STDLIB_CONCURRENCY_TRACING=NO \
+      -DSWIFT_RUNTIME_CRASH_REPORTER_CLIENT=NO \
+      -DSWIFT_STDLIB_INSTALL_PARENT_MODULE_FOR_SHIMS=NO \
+    " \
+    --llvm-cmake-options="\
+      -DCROSS_TOOLCHAIN_FLAGS_LLVM_NATIVE='-DCMAKE_C_COMPILER=clang;-DCMAKE_CXX_COMPILER=clang++' \
+    " \
+    --sccache
+
+  local BUILD_TOOLS_ARGS=(
+    "$OPTIONS_LLVM_BIN"
+    "$OPTIONS_CLANG_BIN"
+    "$OPTIONS_SWIFT_BIN"
+  )
+
+  build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasi" "wasi-wasm32" "wasm32-wasi" "wasmstdlib" "wasi"
+  build_target_toolchain "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1-threads" "wasip1-threads-wasm32" "wasm32-wasip1-threads" "wasmthreadsstdlib" "wasip1"
+
+  rsync -av "$WASI_SYSROOT_PATH/" "$PACKAGING_DIR/wasi-sysroot/"
+
+  build_target_corelibs "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasi" "wasm32-wasi"
+  build_target_corelibs "${BUILD_TOOLS_ARGS[@]}" "wasm32-unknown-wasip1-threads" "wasm32-wasip1-threads"
+}
+
+main "$@"