[Driver][clang-linker-wrapper] Add initial support for OpenMP offloading to generic SPIR-V

This is the first of a series of patches to add support for OpenMP offloading to SPIR-V through liboffload with the first intended target being Intel GPUs. This patch implements the basic driver and `clang-linker-wrapper` work for JIT mode. There are still many missing pieces, so this is not yet usable.

We introduce `spirv64-intel-unknown` as the only currently supported triple. The user-facing argument to enable offloading will be `-fopenmp -fopenmp-targets=spirv64-intel`

Add a new `SPIRVOpenMPToolChain` toolchain based on the existing general SPIR-V toolchain which will call all the required SPIR-V tools as well as add the device RTL as an argument to the linker.

As there is no production quality SPIR-V linker available, manually create an ELF binary containing the offloading image in a way that fits into the existing `liboffload` infrastructure. This ELF will eventually be passed to a runtime plugin that interacts with the Intel GPU runtime.

There is also a small fix to an issue I found when trying to assemble SPIR-V when in text format.

Signed-off-by: Sarnie, Nick <[email protected]>

Author: sarnex

clang/include/clang/Driver/Options.td

@@ -1493,6 +1493,8 @@ def libomptarget_amdgcn_bc_path_EQ : Joined<["--"], "libomptarget-amdgcn-bc-path
   HelpText<"Path to libomptarget-amdgcn bitcode library">, Alias<libomptarget_amdgpu_bc_path_EQ>;
 def libomptarget_nvptx_bc_path_EQ : Joined<["--"], "libomptarget-nvptx-bc-path=">, Group<i_Group>,
   HelpText<"Path to libomptarget-nvptx bitcode library">;
+def libomptarget_spirv_bc_path_EQ : Joined<["--"], "libomptarget-spirv-bc-path=">, Group<i_Group>,
+  HelpText<"Path to libomptarget-spirv bitcode library">;
 def dD : Flag<["-"], "dD">, Group<d_Group>, Visibility<[ClangOption, CC1Option]>,
   HelpText<"Print macro definitions in -E mode in addition to normal output">;
 def dI : Flag<["-"], "dI">, Group<d_Group>, Visibility<[ClangOption, CC1Option]>,

clang/lib/Driver/CMakeLists.txt

@@ -77,6 +77,7 @@ add_clang_library(clangDriver
   ToolChains/RISCVToolchain.cpp
   ToolChains/Solaris.cpp
   ToolChains/SPIRV.cpp
+  ToolChains/SPIRVOpenMP.cpp
   ToolChains/TCE.cpp
   ToolChains/UEFI.cpp
   ToolChains/VEToolchain.cpp

clang/lib/Driver/Driver.cpp

@@ -43,6 +43,7 @@
 #include "ToolChains/PS4CPU.h"
 #include "ToolChains/RISCVToolchain.h"
 #include "ToolChains/SPIRV.h"
+#include "ToolChains/SPIRVOpenMP.h"
 #include "ToolChains/Solaris.h"
 #include "ToolChains/TCE.h"
 #include "ToolChains/UEFI.h"
@@ -166,6 +167,20 @@ getHIPOffloadTargetTriple(const Driver &D, const ArgList &Args) {
   return std::nullopt;
 }
 
+static std::optional<llvm::Triple>
+getSPIRVOffloadTargetTriple(const Driver &D, const ArgList &Args) {
+  if (!Args.hasArg(options::OPT_offload_EQ))
+    return llvm::Triple(
+        "spirv64-intel"); // Only vendor "intel" is currently supported.
+  auto TT = getOffloadTargetTriple(D, Args);
+  if (!TT)
+    return std::nullopt;
+  if ((*TT).isSPIRV() && (*TT).getVendor() == llvm::Triple::Intel)
+    return TT;
+  D.Diag(diag::err_drv_invalid_or_unsupported_offload_target) << TT->str();
+  return std::nullopt;
+}
+
 // static
 std::string Driver::GetResourcesPath(StringRef BinaryPath) {
   // Since the resource directory is embedded in the module hash, it's important
@@ -888,11 +903,12 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
       auto AMDTriple = getHIPOffloadTargetTriple(*this, C.getInputArgs());
       auto NVPTXTriple = getNVIDIAOffloadTargetTriple(*this, C.getInputArgs(),
                                                       HostTC->getTriple());
+      auto SPIRVTriple = getSPIRVOffloadTargetTriple(*this, C.getInputArgs());
 
       // Attempt to deduce the offloading triple from the set of architectures.
-      // We can only correctly deduce NVPTX / AMDGPU triples currently. We need
-      // to temporarily create these toolchains so that we can access tools for
-      // inferring architectures.
+      // We can only correctly deduce NVPTX / AMDGPU / SPIR-V triples currently.
+      // We need to temporarily create these toolchains so that we can access
+      // tools for inferring architectures.
       llvm::DenseSet<StringRef> Archs;
       if (NVPTXTriple) {
         auto TempTC = std::make_unique<toolchains::CudaToolChain>(
@@ -908,7 +924,16 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                  C, C.getArgs(), Action::OFK_OpenMP, &*TempTC, true))
           Archs.insert(Arch);
       }
-      if (!AMDTriple && !NVPTXTriple) {
+
+      if (SPIRVTriple) {
+        auto TempTC = std::make_unique<toolchains::SPIRVOpenMPToolChain>(
+            *this, *SPIRVTriple, *HostTC, C.getInputArgs());
+        for (StringRef Arch : getOffloadArchs(
+                 C, C.getArgs(), Action::OFK_OpenMP, &*TempTC, true))
+          Archs.insert(Arch);
+      }
+
+      if (!AMDTriple && !NVPTXTriple && !SPIRVTriple) {
         for (StringRef Arch :
              getOffloadArchs(C, C.getArgs(), Action::OFK_OpenMP, nullptr, true))
           Archs.insert(Arch);
@@ -922,6 +947,8 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
                    IsAMDOffloadArch(StringToOffloadArch(
                        getProcessorFromTargetID(*AMDTriple, Arch)))) {
           DerivedArchs[AMDTriple->getTriple()].insert(Arch);
+        } else if (SPIRVTriple && Arch == (*SPIRVTriple).str()) {
+          DerivedArchs[Arch].insert(Arch);
         } else {
           Diag(clang::diag::err_drv_failed_to_deduce_target_from_arch) << Arch;
           return;
@@ -962,7 +989,7 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
         const ToolChain *TC;
         // Device toolchains have to be selected differently. They pair host
         // and device in their implementation.
-        if (TT.isNVPTX() || TT.isAMDGCN()) {
+        if (TT.isNVPTX() || TT.isAMDGCN() || TT.isSPIRV()) {
           const ToolChain *HostTC =
               C.getSingleOffloadToolChain<Action::OFK_Host>();
           assert(HostTC && "Host toolchain should be always defined.");
@@ -975,6 +1002,9 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
             else if (TT.isAMDGCN())
               DeviceTC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(
                   *this, TT, *HostTC, C.getInputArgs());
+            else if (TT.isSPIRV())
+              DeviceTC = std::make_unique<toolchains::SPIRVOpenMPToolChain>(
+                  *this, TT, *HostTC, C.getInputArgs());
             else
               assert(DeviceTC && "Device toolchain not defined.");
           }

clang/lib/Driver/ToolChains/CommonArgs.cpp

@@ -2829,10 +2829,13 @@ void tools::addOpenMPDeviceRTL(const Driver &D,
     LibraryPaths.emplace_back(LibPath);
 
   OptSpecifier LibomptargetBCPathOpt =
-      Triple.isAMDGCN() ? options::OPT_libomptarget_amdgpu_bc_path_EQ
-                        : options::OPT_libomptarget_nvptx_bc_path_EQ;
+      Triple.isAMDGCN()  ? options::OPT_libomptarget_amdgpu_bc_path_EQ
+      : Triple.isNVPTX() ? options::OPT_libomptarget_nvptx_bc_path_EQ
+                         : options::OPT_libomptarget_spirv_bc_path_EQ;
 
-  StringRef ArchPrefix = Triple.isAMDGCN() ? "amdgpu" : "nvptx";
+  StringRef ArchPrefix = Triple.isAMDGCN()  ? "amdgpu"
+                         : Triple.isNVPTX() ? "nvptx"
+                                            : "spirv64";
   std::string LibOmpTargetName =
       ("libomptarget-" + ArchPrefix + "-" + BitcodeSuffix + ".bc").str();
 

clang/lib/Driver/ToolChains/SPIRV.cpp

@@ -28,8 +28,11 @@ void SPIRV::constructTranslateCommand(Compilation &C, const Tool &T,
 
   if (Input.getType() == types::TY_PP_Asm)
     CmdArgs.push_back("-to-binary");
+
+  // The text output from spirv-dis is not in the format expected
+  // by llvm-spirv, so use the text output from llvm-spirv.
   if (Output.getType() == types::TY_PP_Asm)
-    CmdArgs.push_back("--spirv-tools-dis");
+    CmdArgs.push_back("--spirv-text");
 
   CmdArgs.append({"-o", Output.getFilename()});
 

clang/lib/Driver/ToolChains/SPIRV.h

@@ -52,7 +52,7 @@ class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY SPIRVToolChain final : public ToolChain {
+class LLVM_LIBRARY_VISIBILITY SPIRVToolChain : public ToolChain {
   mutable std::unique_ptr<Tool> Translator;
 
 public:

clang/lib/Driver/ToolChains/SPIRVOpenMP.cpp

@@ -0,0 +1,36 @@
+//==- SPIRVOpenMP.cpp - SPIR-V OpenMP Tool Implementations --------*- C++ -*==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//==------------------------------------------------------------------------==//
+#include "SPIRVOpenMP.h"
+#include "CommonArgs.h"
+
+using namespace clang::driver;
+using namespace clang::driver::toolchains;
+using namespace clang::driver::tools;
+using namespace llvm::opt;
+
+namespace clang::driver::toolchains {
+SPIRVOpenMPToolChain::SPIRVOpenMPToolChain(const Driver &D,
+                                           const llvm::Triple &Triple,
+                                           const ToolChain &HostToolchain,
+                                           const ArgList &Args)
+    : SPIRVToolChain(D, Triple, Args), HostTC(HostToolchain) {}
+
+void SPIRVOpenMPToolChain::addClangTargetOptions(
+    const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
+    Action::OffloadKind DeviceOffloadingKind) const {
+
+  if (DeviceOffloadingKind != Action::OFK_OpenMP)
+    return;
+
+  if (DriverArgs.hasArg(options::OPT_nogpulib))
+    return;
+  Twine GpuArch = getTriple().getArchName() + "-" + getTriple().getVendorName();
+  addOpenMPDeviceRTL(getDriver(), DriverArgs, CC1Args, GpuArch.str(),
+                     getTriple(), HostTC);
+}
+} // namespace clang::driver::toolchains

clang/lib/Driver/ToolChains/SPIRVOpenMP.h

@@ -0,0 +1,29 @@
+//===--- SPIRVOpenMP.h - SPIR-V OpenMP Tool Implementations ------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_SPIRV_OPENMP_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_SPIRV_OPENMP_H
+
+#include "SPIRV.h"
+#include "clang/Driver/Tool.h"
+#include "clang/Driver/ToolChain.h"
+
+namespace clang::driver::toolchains {
+class LLVM_LIBRARY_VISIBILITY SPIRVOpenMPToolChain : public SPIRVToolChain {
+public:
+  SPIRVOpenMPToolChain(const Driver &D, const llvm::Triple &Triple,
+                       const ToolChain &HostTC, const llvm::opt::ArgList &Args);
+
+  void addClangTargetOptions(
+      const llvm::opt::ArgList &DriverArgs, llvm::opt::ArgStringList &CC1Args,
+      Action::OffloadKind DeviceOffloadingKind) const override;
+
+  const ToolChain &HostTC;
+};
+} // namespace clang::driver::toolchains
+#endif

clang/lib/Frontend/CompilerInvocation.cpp

@@ -4256,6 +4256,7 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
 
       if (TT.getArch() == llvm::Triple::UnknownArch ||
           !(TT.getArch() == llvm::Triple::aarch64 || TT.isPPC() ||
+            TT.getArch() == llvm::Triple::spirv64 ||
             TT.getArch() == llvm::Triple::systemz ||
             TT.getArch() == llvm::Triple::nvptx ||
             TT.getArch() == llvm::Triple::nvptx64 ||