Switch native library compiler from GCC 14 to Clang 21 (elastic#145681)

This PR switches the compiler for the native simdvec library (libvec) from GCC 14 to Clang 21
for all targets (Darwin, Linux x64, Linux aarch64).

Benchmarking showed Clang 21 produces 8-12% faster AVX-512 code than GCC 14 on x64, due to better register allocation (GCC generates unnecessary vmovdqa register-to-register copies in unrolled loops). On ARM, performance is identical between compilers.

Author: ldematte

libs/native/libraries/build.gradle

@@ -19,7 +19,7 @@ configurations {
 }
 
 var zstdVersion = "1.5.7"
-var vecVersion = "1.0.79"
+var vecVersion = "1.0.83"
 
 repositories {
   exclusiveContent {

libs/simdvec/native/Dockerfile.cross-toolchain

@@ -8,37 +8,36 @@
 #
 
 # Cross-compilation toolchain image for libvec.
-# Contains clang-18+lld-18 (Darwin target) and gcc-14 (Linux targets).
+# Contains clang-21 (all targets: Darwin, Linux x64, Linux aarch64).
 # No macOS SDK required — Darwin builds use a minimal C++ std library
 # (tinystd) bundled in the source tree, with clang's freestanding headers.
 #
 # To build and push (see also build_cross_toolchain_image.sh):
 #   docker build \
 #     -f Dockerfile.cross-toolchain \
-#     -t docker.elastic.co/es-dev/es-native-cross-toolchain:1 .
-#   docker push docker.elastic.co/es-dev/es-native-cross-toolchain:1
+#     -t docker.elastic.co/es-dev/es-native-cross-toolchain:2 .
+#   docker push docker.elastic.co/es-dev/es-native-cross-toolchain:2
 
 FROM debian:trixie-slim
 
-# clang-18 and lld-18 from the official LLVM apt repository.
+# clang-21 from the official LLVM apt repository.
+# libstdc++-14-dev-arm64-cross provides the aarch64 sysroot (C++ headers + libs)
+# for cross-compiling Linux aarch64 from x64.
 RUN apt-get update && apt-get install -y --no-install-recommends \
         ca-certificates \
         gnupg \
         wget \
     && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \
         | gpg --dearmor > /usr/share/keyrings/llvm-archive-keyring.gpg \
     && echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] \
-        https://apt.llvm.org/trixie/ llvm-toolchain-trixie-18 main" \
-        > /etc/apt/sources.list.d/llvm.list \
+        https://apt.llvm.org/trixie/ llvm-toolchain-trixie-21 main" \
+        > /etc/apt/sources.list.d/llvm-21.list \
     && rm -rf /var/lib/apt/lists/*
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
         make \
-        clang-18 \
-        lld-18 \
-        llvm-18 \
-        gcc-14 \
-        g++-14 \
-        gcc-14-aarch64-linux-gnu \
-        g++-14-aarch64-linux-gnu \
+        clang-21 \
+        lld-21 \
+        llvm-21 \
+        libstdc++-14-dev-arm64-cross \
     && rm -rf /var/lib/apt/lists/*

libs/simdvec/native/Makefile

@@ -29,14 +29,14 @@ MARCH_AMD64_TIER2   = icelake-client
 # Compilers
 # Darwin cross-compilation: -nostdinc blocks host (glibc) headers, then we
 # add our minimal C++ std headers (tinystd) and clang builtins.
-CLANG_RESOURCE = $(shell clang++-18 -print-resource-dir)
-CLANG       = clang++-18 --target=arm64-apple-macos14 -Winline -nostdinc \
+CLANG_RESOURCE = $(shell clang++-21 -print-resource-dir)
+CLANG       = clang++-21 --target=arm64-apple-macos14 -Winline -nostdinc \
                 -isystem src/vec/headers/darwin/tinystd \
                 -isystem $(CLANG_RESOURCE)/include
-CLANG_LINK  = clang++-18 --target=arm64-apple-macos14 -fuse-ld=lld -nostdlib -Wl,-undefined,dynamic_lookup
-CLANG_STRIP = llvm-strip-18 --strip-debug
-GCC_ARM     = aarch64-linux-gnu-g++-14 -Winline
-GCC_X64     = g++-14 -Winline
+CLANG_LINK  = clang++-21 --target=arm64-apple-macos14 -fuse-ld=lld -nostdlib -Wl,-undefined,dynamic_lookup
+CLANG_STRIP = llvm-strip-21 --strip-debug
+CXX_ARM     = clang++-21 --target=aarch64-linux-gnu -Winline
+CXX_X64     = clang++-21 -Winline
 
 # Sources split by tier
 SRC_AARCH64_TIER1 = $(filter-out %_2.cpp, $(wildcard src/vec/c/aarch64/*.cpp))
@@ -66,7 +66,7 @@ SO_X64 = $(OUT_DIR_AMD64)/libvec.so
 
 # --- local: build for the current platform using the native compiler ---
 # On macOS, overrides CLANG/CLANG_LINK to use the system clang (SDK available natively).
-# On Linux, the GCC variables already use the native compiler.
+# On Linux aarch64, overrides CXX_ARM to drop the cross-compilation target.
 
 .PHONY: all clean local install
 
@@ -80,6 +80,7 @@ ifeq ($(shell uname -s),Darwin)
 else ifeq ($(shell uname -m),aarch64)
   LOCAL_PLATFORM = linux-aarch64
   LOCAL_OUTPUT   = $(SO_ARM)
+  local: CXX_ARM = clang++-21 -Winline
   local: $(SO_ARM)
 else
   LOCAL_PLATFORM = linux-x64
@@ -112,26 +113,26 @@ $(DYLIB): $(OBJ_DARWIN_TIER1) $(OBJ_DARWIN_TIER2) | $(OUT_DIR_AARCH64)
 # --- linux-aarch64 ---
 
 $(OBJ_LIN_ARM_TIER1): $(OBJ_LIN_ARM)/%.o: src/vec/c/aarch64/%.cpp | $(OBJ_LIN_ARM)
-	$(GCC_ARM) $(CXXFLAGS) -march=$(MARCH_AARCH64_TIER1) $(HEADERS) -c $< -o $@
+	$(CXX_ARM) $(CXXFLAGS) -march=$(MARCH_AARCH64_TIER1) $(HEADERS) -c $< -o $@
 
 $(OBJ_LIN_ARM_TIER2): $(OBJ_LIN_ARM)/%.o: src/vec/c/aarch64/%.cpp | $(OBJ_LIN_ARM)
-	$(GCC_ARM) $(CXXFLAGS) -march=$(MARCH_AARCH64_TIER2) $(HEADERS) -c $< -o $@
+	$(CXX_ARM) $(CXXFLAGS) -march=$(MARCH_AARCH64_TIER2) $(HEADERS) -c $< -o $@
 
 $(SO_ARM): $(OBJ_LIN_ARM_TIER1) $(OBJ_LIN_ARM_TIER2) | $(OUT_DIR_AARCH64)
-	$(GCC_ARM) -shared $^ -o $@
-	aarch64-linux-gnu-strip --strip-unneeded $@
+	$(CXX_ARM) -fuse-ld=lld -shared $^ -o $@
+	llvm-strip-21 --strip-unneeded $@
 
 # --- linux-x64 ---
 
 $(OBJ_LIN_X64_TIER1): $(OBJ_LIN_X64)/%.o: src/vec/c/amd64/%.cpp | $(OBJ_LIN_X64)
-	$(GCC_X64) $(CXXFLAGS) -march=$(MARCH_AMD64_TIER1) $(HEADERS) -c $< -o $@
+	$(CXX_X64) $(CXXFLAGS) -march=$(MARCH_AMD64_TIER1) $(HEADERS) -c $< -o $@
 
 $(OBJ_LIN_X64_TIER2): $(OBJ_LIN_X64)/%.o: src/vec/c/amd64/%.cpp | $(OBJ_LIN_X64)
-	$(GCC_X64) $(CXXFLAGS) -march=$(MARCH_AMD64_TIER2) $(HEADERS) -c $< -o $@
+	$(CXX_X64) $(CXXFLAGS) -march=$(MARCH_AMD64_TIER2) $(HEADERS) -c $< -o $@
 
 $(SO_X64): $(OBJ_LIN_X64_TIER1) $(OBJ_LIN_X64_TIER2) | $(OUT_DIR_AMD64)
-	$(GCC_X64) -shared $^ -o $@
-	strip --strip-unneeded $@
+	$(CXX_X64) -shared $^ -o $@
+	llvm-strip-21 --strip-unneeded $@
 
 # --- directories ---
 

libs/simdvec/native/build_cross_toolchain_image.sh

@@ -27,7 +27,7 @@ case "${1:-}" in
   *)       echo "Usage: $0 [--local]" >&2; exit 1 ;;
 esac
 
-VERSION=1
+VERSION=2
 HOST=docker.elastic.co
 REPOSITORY=elasticsearch-infra/es-native-cross-toolchain
 IMAGE=$HOST/$REPOSITORY:$VERSION

libs/simdvec/native/publish_vec_binaries.sh

@@ -17,7 +17,7 @@
 
 set -euo pipefail
 
-VERSION="1.0.79"
+VERSION="1.0.83"
 
 LOCAL=false
 FORCE_UPLOAD=false
@@ -39,7 +39,7 @@ if [ "$UPLOAD" = true ] && [ -z "${ARTIFACTORY_API_KEY:-}" ]; then
   exit 1;
 fi
 
-TOOLCHAIN_IMAGE="docker.elastic.co/elasticsearch-infra/es-native-cross-toolchain:1"
+TOOLCHAIN_IMAGE="docker.elastic.co/elasticsearch-infra/es-native-cross-toolchain:2"
 if [ "$LOCAL" = true ]; then
   TOOLCHAIN_IMAGE="es-native-cross-toolchain:local"
 fi