[SYCL] Embed bfloat16 devicelib into executable if necessary (#16729)

Currently, sycl bfloat16 conversion functions are implemented in 2
devicelib spvs(fallback version and native version).
The native version targets for any platform which supports
"cl_intel_bfloat16_conversions" extension and fallback version is used
for all other platforms.
SYCL runtime will select the bfloat16 spvs during execution time by
checking bfloat16 extension.
The design requires us to ship 2 spv files together with sycl runtime
which some users may dislike.
The PR uses sycl dynamic library mechanism to re-implement this
behavior. These 2 bfloat16 lib files are regarded as dynamic library and
embedded to final executable, so we don't need to ship any bfloat16 spv
libs.
The PR consists following changes:
1. Driver: pass the devicelib file location to sycl-post-link tool.
2. sycl-post-link: analyze user's device image to see whether bfloat16
devicelib functions are used. If yes, add the 2 bfloat16 devicelib files
as "required" dynamic library. All required bfloat16 devicelib functions
are treated as "imported" symbols in user's device image and all
functions in bfloat16 devicelib are "exported" symbols.
3. Sycl runtime will load and link the required bfloat16 devicelib image
and resolve the imported symbols.

Fallback and native version of bfloat16 devicelib files have exactly
same exported functions, we add a new
metadata("SYCL_DEVICELIB_BF16_TYPE") to indicate the version in them.
SYCL runtime will check cl_intel_bfloat16_conversions extension and this
metadata to decide which version will be linked.

---------

Signed-off-by: jinge90 <[email protected]>
Co-authored-by: Steffen Larsen <[email protected]>
Co-authored-by: Alexey Sachkov <[email protected]>

Author: 3 people

clang/lib/Driver/ToolChains/Clang.cpp

@@ -11022,6 +11022,7 @@ static void getNonTripleBasedSYCLPostLinkOpts(const ToolChain &TC,
                                               const JobAction &JA,
                                               const llvm::opt::ArgList &TCArgs,
                                               ArgStringList &PostLinkArgs) {
+
   // See if device code splitting is requested
   if (Arg *A = TCArgs.getLastArg(options::OPT_fsycl_device_code_split_EQ)) {
     auto CodeSplitValue = StringRef(A->getValue());
@@ -11047,6 +11048,25 @@ static void getNonTripleBasedSYCLPostLinkOpts(const ToolChain &TC,
 
   if (allowDeviceImageDependencies(TCArgs))
     addArgs(PostLinkArgs, TCArgs, {"-allow-device-image-dependencies"});
+
+  // For bfloat16 conversions LLVM IR devicelib, we only need to embed it
+  // when non-AOT compilation is used.
+  if (TC.getTriple().isSPIROrSPIRV() && !TC.getTriple().isSPIRAOT()) {
+    SYCLInstallationDetector SYCLInstall(TC.getDriver());
+    SmallVector<SmallString<128>, 4> DeviceLibLocCandidates;
+    SmallString<128> NativeBfloat16Name("libsycl-native-bfloat16.bc");
+    SYCLInstall.getSYCLDeviceLibPath(DeviceLibLocCandidates);
+    for (const auto &DeviceLibLoc : DeviceLibLocCandidates) {
+      SmallString<128> FullLibName(DeviceLibLoc);
+      llvm::sys::path::append(FullLibName, NativeBfloat16Name);
+      if (llvm::sys::fs::exists(FullLibName)) {
+        SmallString<128> SYCLDeviceLibDir("--device-lib-dir=");
+        SYCLDeviceLibDir += DeviceLibLoc.str();
+        addArgs(PostLinkArgs, TCArgs, {SYCLDeviceLibDir.str()});
+        break;
+      }
+    }
+  }
 }
 
 // On Intel targets we don't need non-kernel functions as entry points,

clang/test/Driver/Inputs/SYCL/lib/libsycl-crt.new.o

@@ -116,3 +116,17 @@
 // Error handling when --linker-path is not provided for clang-linker-wrapper
 // RUN: not clang-linker-wrapper 2>&1 | FileCheck --check-prefix=LINKER-PATH-NOT-PROVIDED %s
 // LINKER-PATH-NOT-PROVIDED: linker path missing, must pass 'linker-path'
+
+/// Check --device-lib-dir for sycl-post-link tool
+// ------
+// Generate .o file as linker wrapper input.
+
+// RUN: %clang %s -fsycl -fsycl-targets=spir64-unknown-unknown -c --offload-new-driver -o %t5.o
+//
+// Run clang-linker-wrapper test
+//
+// RUN: clang-linker-wrapper -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-pc-windows-msvc" "--linker-path=/usr/bin/ld" "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t5.o -sycl-device-libraries=libsycl-crt.new.o -sycl-device-library-location=%S/Inputs/SYCL/lib --dry-run 2>&1 | FileCheck -check-prefix=CHK-CMDS-DEVICE-LIB-DIR %s
+// CHK-CMDS-DEVICE-LIB-DIR: "{{.*}}spirv-to-ir-wrapper.exe" {{.*}} --llvm-spirv-opts --spirv-preserve-auxdata --spirv-target-env=SPV-IR --spirv-builtin-format=global
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link.exe" {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link.exe" -only-needed {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}sycl-post-link.exe"{{.*}} --device-lib-dir={{.*}}/Inputs/SYCL/lib {{.*}} SYCL_POST_LINK_OPTIONS {{.*}}

clang/test/Driver/linker-wrapper-sycl-win.cpp

@@ -194,3 +194,15 @@
 // Error handling when --linker-path is not provided for clang-linker-wrapper
 // RUN: not clang-linker-wrapper 2>&1 | FileCheck --check-prefix=LINKER-PATH-NOT-PROVIDED %s
 // LINKER-PATH-NOT-PROVIDED: linker path missing, must pass 'linker-path'
+
+/// check for --device-lib-dir options for sycl-post-link.
+// -------
+// Generate .o file as linker wrapper input.
+//
+// RUN: %clang %s -fsycl -fsycl-targets=spir64-unknown-unknown -c --offload-new-driver -o %t5.o
+//
+// RUN: clang-linker-wrapper -sycl-post-link-options="SYCL_POST_LINK_OPTIONS" -llvm-spirv-options="LLVM_SPIRV_OPTIONS" "--host-triple=x86_64-unknown-linux-gnu" "--linker-path=/usr/bin/ld" "--" HOST_LINKER_FLAGS "-dynamic-linker" HOST_DYN_LIB "-o" "a.out" HOST_LIB_PATH HOST_STAT_LIB %t5.o -sycl-device-libraries=libsycl-crt.new.o -sycl-device-library-location=%S/Inputs/SYCL/lib --dry-run 2>&1 | FileCheck -check-prefix=CHK-CMDS-DEVICE-LIB-DIR %s
+// CHK-CMDS-DEVICE-LIB-DIR: "{{.*}}spirv-to-ir-wrapper" {{.*}} --llvm-spirv-opts --spirv-preserve-auxdata --spirv-target-env=SPV-IR --spirv-builtin-format=global
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link" {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}llvm-link" -only-needed {{.*}} --suppress-warnings
+// CHK-CMDS-DEVICE-LIB-DIR-NEXT: "{{.*}}sycl-post-link"{{.*}} --device-lib-dir={{.*}}/Inputs/SYCL/lib {{.*}} SYCL_POST_LINK_OPTIONS {{.*}}

clang/test/Driver/linker-wrapper-sycl.cpp

@@ -1,9 +1,9 @@
 // REQUIRES: system-windows
 /// Verify same set of sycl-post-link options generated for old and new offloading model
 // RUN: %clangxx -### --target=x86_64-pc-windows-msvc -fsycl \
-// RUN:          -Xdevice-post-link -O0 %s 2>&1 \
+// RUN:          -Xdevice-post-link -O0 %s --sysroot=%S/Inputs/SYCL 2>&1 \
 // RUN:   | FileCheck -check-prefix OPTIONS_POSTLINK_JIT_OLD %s
-// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
+// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "--device-lib-dir={{.*}}" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
 // -------
 // Generate .o file as linker wrapper input.
 //

clang/test/Driver/sycl-post-link-options-win.cpp

@@ -1,9 +1,9 @@
 // REQUIRES: system-linux
 /// Verify same set of sycl-post-link options generated for old and new offloading model
 // RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl -### \
-// RUN:          --no-offload-new-driver -Xdevice-post-link -O0 %s 2>&1 \
+// RUN:          --no-offload-new-driver -Xdevice-post-link -O0 %s --sysroot=%S/Inputs/SYCL 2>&1 \
 // RUN:   | FileCheck -check-prefix OPTIONS_POSTLINK_JIT_OLD %s
-// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
+// OPTIONS_POSTLINK_JIT_OLD: sycl-post-link{{.*}} "-O2" "-device-globals" "--device-lib-dir={{.*}}" "-properties" "-spec-const=native" "-split=auto" "-emit-only-kernels-as-entry-points" "-emit-param-info" "-symbols" "-emit-exported-symbols" "-emit-imported-symbols" "-split-esimd" "-lower-esimd" "-O0"
 //
 // Generate .o file as linker wrapper input.
 //

clang/test/Driver/sycl-post-link-options.cpp

@@ -726,6 +726,16 @@ runSYCLPostLinkTool(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   SmallVector<StringRef, 8> CmdArgs;
   CmdArgs.push_back(*SYCLPostLinkPath);
   const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
+  Arg *SYCLDeviceLibLoc = Args.getLastArg(OPT_sycl_device_library_location_EQ);
+  if (SYCLDeviceLibLoc && !Triple.isSPIRAOT()) {
+    std::string SYCLDeviceLibLocParam = SYCLDeviceLibLoc->getValue();
+    std::string BF16DeviceLibLoc =
+        SYCLDeviceLibLocParam + "/libsycl-native-bfloat16.bc";
+    if (llvm::sys::fs::exists(BF16DeviceLibLoc)) {
+      SYCLDeviceLibLocParam = "--device-lib-dir=" + SYCLDeviceLibLocParam;
+      CmdArgs.push_back(Args.MakeArgString(StringRef(SYCLDeviceLibLocParam)));
+    }
+  }
   getTripleBasedSYCLPostLinkOpts(Args, CmdArgs, Triple);
   StringRef SYCLPostLinkOptions;
   if (Arg *A = Args.getLastArg(OPT_sycl_post_link_options_EQ))

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

@@ -12,6 +12,7 @@
 
 #include "llvm/ADT/SetVector.h"
 #include "llvm/SYCLLowerIR/ModuleSplitter.h"
+#include "llvm/SYCLLowerIR/SYCLDeviceLibReqMask.h"
 #include "llvm/Support/PropertySetIO.h"
 #include <string>
 namespace llvm {
@@ -34,6 +35,9 @@ bool isModuleUsingTsan(const Module &M);
 using PropSetRegTy = llvm::util::PropertySetRegistry;
 using EntryPointSet = SetVector<Function *>;
 
+PropSetRegTy computeDeviceLibProperties(const Module &M,
+                                        const std::string &SYCLDeviceLibName);
+
 PropSetRegTy computeModuleProperties(const Module &M,
                                      const EntryPointSet &EntryPoints,
                                      const GlobalBinImageProps &GlobProps);

llvm/include/llvm/SYCLLowerIR/ComputeModuleRuntimeInfo.h

@@ -21,6 +21,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/PropertySetIO.h"
 
+#include <array>
 #include <memory>
 #include <optional>
 #include <string>
@@ -38,6 +39,8 @@ class OptionCategory;
 namespace module_split {
 
 constexpr char SYCL_ESIMD_SPLIT_MD_NAME[] = "sycl-esimd-split-status";
+constexpr std::array<const char *, 2> SYCLDeviceLibs = {
+    "libsycl-fallback-bfloat16.bc", "libsycl-native-bfloat16.bc"};
 
 extern cl::OptionCategory &getModuleSplitCategory();
 
@@ -129,6 +132,7 @@ class ModuleDesc {
   std::unique_ptr<Module> M;
   EntryPointGroup EntryPoints;
   bool IsTopLevel = false;
+  bool IsSYCLDeviceLib = false;
   mutable std::optional<SYCLDeviceRequirements> Reqs;
 
 public:
@@ -140,7 +144,16 @@ class ModuleDesc {
   Properties Props;
 
   ModuleDesc(std::unique_ptr<Module> &&M, StringRef Name = "TOP-LEVEL")
-      : M(std::move(M)), IsTopLevel(true), Name(Name) {}
+      : M(std::move(M)), IsTopLevel(true), Name(Name) {
+    // DeviceLib module doesn't include any entry point,it can be constructed
+    // using ctor without any entry point related parameter.
+    for (auto Fn : SYCLDeviceLibs) {
+      if (StringRef(Fn) == Name) {
+        IsSYCLDeviceLib = true;
+        break;
+      }
+    }
+  }
 
   ModuleDesc(std::unique_ptr<Module> &&M, EntryPointGroup &&EntryPoints,
              const Properties &Props)
@@ -166,6 +179,7 @@ class ModuleDesc {
 
   bool isESIMD() const { return EntryPoints.isEsimd(); }
   bool isSYCL() const { return EntryPoints.isSycl(); }
+  bool isSYCLDeviceLib() const { return IsSYCLDeviceLib; }
 
   const EntryPointSet &entries() const { return EntryPoints.Functions; }
   const EntryPointGroup &getEntryPointGroup() const { return EntryPoints; }

llvm/include/llvm/SYCLLowerIR/ModuleSplitter.h

@@ -20,11 +20,14 @@
 
 namespace llvm {
 
+class Function;
 class Module;
 
 // DeviceLibExt is shared between sycl-post-link tool and sycl runtime.
 // If any change is made here, need to sync with DeviceLibExt definition
 // in sycl/source/detail/program_manager/program_manager.hpp
+// TODO: clear all these DeviceLibExt defs when begin to remove sycl
+// devicelib online link path.
 enum class DeviceLibExt : std::uint32_t {
   cl_intel_devicelib_assert,
   cl_intel_devicelib_math,
@@ -39,5 +42,6 @@ enum class DeviceLibExt : std::uint32_t {
 };
 
 uint32_t getSYCLDeviceLibReqMask(const Module &M);
-
+bool isSYCLDeviceLibBF16Used(const Module &M);
+bool isBF16DeviceLibFuncDecl(const Function &F);
 } // namespace llvm