[Driver][SYCL] Enable --offload-arch support for SYCL offloading.

Author: srividya-sundaram

clang/include/clang/Basic/Cuda.h

@@ -106,6 +106,8 @@ enum class OffloadArch {
   GFX90a,
   GFX90c,
   GFX9_4_GENERIC,
+  GFX940,
+  GFX941,  
   GFX942,
   GFX950,
   GFX10_1_GENERIC,

clang/include/clang/Basic/DiagnosticDriverKinds.td

@@ -843,4 +843,14 @@ def warn_missing_include_dirs : Warning<
 
 def err_drv_malformed_warning_suppression_mapping : Error<
   "failed to process suppression mapping file '%0': %1">;
+
+def err_drv_sycl_offload_arch_missing_value : Error<
+  "must pass in an explicit cpu or gpu architecture to '--offload-arch'">;
+
+def err_drv_invalid_sycl_target : Error<"SYCL target is invalid: '%0'">;
+
+def warn_drv_sycl_offload_target_duplicate : Warning<
+  "SYCL offloading target '%0' is similar to target '%1' already specified; "
+  "will be ignored">, InGroup<SyclTarget>;
+
 }

clang/include/clang/Basic/DiagnosticGroups.td

@@ -1628,3 +1628,7 @@ def ExplicitSpecializationStorageClass : DiagGroup<"explicit-specialization-stor
 
 // A warning for options that enable a feature that is not yet complete
 def ExperimentalOption : DiagGroup<"experimental-option">;
+
+// SYCL Warnings
+def SyclTarget : DiagGroup<"sycl-target">;
+

clang/include/clang/Driver/Driver.h

@@ -846,6 +846,23 @@ class Driver {
   /// Compute the default -fmodule-cache-path.
   /// \return True if the system provides a default cache directory.
   static bool getDefaultModuleCachePath(SmallVectorImpl<char> &Result);
+
+  /// Vector of Macros that need to be added to the Host compilation in a
+  /// SYCL based offloading scenario.  These macros are gathered during
+  /// construction of the device compilations.
+  mutable std::vector<std::string> SYCLTargetMacroArgs;
+
+  /// addSYCLTargetMacroArg - Add the given macro to the vector of args to be
+  /// added to the host compilation step.
+  void addSYCLTargetMacroArg(const llvm::opt::ArgList &Args,
+    StringRef Macro) const {
+    SYCLTargetMacroArgs.push_back(Args.MakeArgString(Macro));
+  }
+
+  /// getSYCLTargetMacroArgs - return the previously gathered macro target args.
+  llvm::ArrayRef<std::string> getSYCLTargetMacroArgs() const {
+    return SYCLTargetMacroArgs;
+  }
 };
 
 /// \return True if the last defined optimization level is -Ofast.

clang/lib/Basic/Cuda.cpp

@@ -124,6 +124,8 @@ static const OffloadArchToStringMap arch_names[] = {
     GFX(90a),  // gfx90a
     GFX(90c),  // gfx90c
     {OffloadArch::GFX9_4_GENERIC, "gfx9-4-generic", "compute_amdgcn"},
+    GFX(940),  // gfx940
+    GFX(941),  // gfx941    
     GFX(942),  // gfx942
     GFX(950),  // gfx950
     {OffloadArch::GFX10_1_GENERIC, "gfx10-1-generic", "compute_amdgcn"},

clang/lib/Driver/Driver.cpp

@@ -833,10 +833,14 @@ Driver::OpenMPRuntimeKind Driver::getOpenMPRuntime(const ArgList &Args) const {
 
 static llvm::Triple getSYCLDeviceTriple(StringRef TargetArch) {
   SmallVector<StringRef, 5> SYCLAlias = {"spir", "spir64", "spirv", "spirv32",
-                                         "spirv64"};
+                                         "spirv64", "spir64_x86_64",
+                                         "spir64_gen", "nvptx64"};
   if (llvm::is_contained(SYCLAlias, TargetArch)) {
     llvm::Triple TargetTriple;
     TargetTriple.setArchName(TargetArch);
+    // Return the full SYCL target triple string for NVidia GPU targets.
+    if (TargetTriple.getArch() == llvm::Triple::nvptx64)
+      return llvm::Triple("nvptx64-nvidia-cuda");
     TargetTriple.setVendor(llvm::Triple::UnknownVendor);
     TargetTriple.setOS(llvm::Triple::UnknownOS);
     return TargetTriple;
@@ -846,16 +850,25 @@ static llvm::Triple getSYCLDeviceTriple(StringRef TargetArch) {
 
 static bool addSYCLDefaultTriple(Compilation &C,
                                  SmallVectorImpl<llvm::Triple> &SYCLTriples) {
+
+  llvm::Triple DefaultTriple = getSYCLDeviceTriple(
+      C.getDefaultToolChain().getTriple().isArch32Bit() ? "spirv32"
+                                                        : "spirv64");
+  for (const auto &SYCLTriple : SYCLTriples) {
+    if (SYCLTriple == DefaultTriple)
+      return false;
+    // If we encounter a known non-spir* target, do not add the default triple.
+    if (SYCLTriple.isNVPTX() || SYCLTriple.isAMDGCN())
+      return false;
+    if(SYCLTriple.isSPIRAOT())
+      return false;
+  }
   // Check current set of triples to see if the default has already been set.
   for (const auto &SYCLTriple : SYCLTriples) {
     if (SYCLTriple.getSubArch() == llvm::Triple::NoSubArch &&
         SYCLTriple.isSPIROrSPIRV())
       return false;
   }
-  // Add the default triple as it was not found.
-  llvm::Triple DefaultTriple = getSYCLDeviceTriple(
-      C.getDefaultToolChain().getTriple().isArch32Bit() ? "spirv32"
-                                                        : "spirv64");
   SYCLTriples.insert(SYCLTriples.begin(), DefaultTriple);
   return true;
 }
@@ -1066,19 +1079,119 @@ void Driver::CreateOffloadingDeviceToolChains(Compilation &C,
   // -ffreestanding cannot be used with -fsycl
   argSYCLIncompatible(options::OPT_ffreestanding);
 
+  // Map of SYCL target triple strings to their corresponding target archs.
+  // Example: spir64_x86_64 --> SKYLAKEAVX512
+  llvm::StringMap<llvm::DenseSet<StringRef>> DerivedArchs;
+  llvm::StringMap<StringRef> FoundNormalizedTriples;
   llvm::SmallVector<llvm::Triple, 4> UniqueSYCLTriplesVec;
-
+  // StringSet to contain SYCL target triples.
+  llvm::StringSet<> SYCLTriples;  
+  // If the user specified --offload-arch, deduce the offloading
+  // target triple(s) from the set of architecture(s).
+  // Create a toolchain for each valid triple.
+  // We do not support SYCL offloading if any of the inputs is a
+  // .cu (for CUDA type) or .hip (for HIP type) file.
   if (IsSYCL) {
-    addSYCLDefaultTriple(C, UniqueSYCLTriplesVec);
+    if(C.getInputArgs().hasArg(options::OPT_offload_arch_EQ) && !IsHIP &&
+    !IsCuda) {
+
+      const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
+      auto AMDTriple = getHIPOffloadTargetTriple(*this, C.getInputArgs());
+      auto NVPTXTriple = getNVIDIAOffloadTargetTriple(*this, C.getInputArgs(),
+                                                      HostTC->getTriple());
+
+      // Attempt to deduce the offloading triple from the set of architectures.
+      // We need to temporarily create these toolchains so that we can access
+      // tools for inferring architectures.
+      llvm::DenseSet<StringRef> Archs;
+      if (NVPTXTriple) {
+        auto TempTC = std::make_unique<toolchains::CudaToolChain>(
+            *this, *NVPTXTriple, *HostTC, C.getInputArgs());
+        for (StringRef Arch :
+            getOffloadArchs(C, C.getArgs(), Action::OFK_SYCL, &*TempTC, true))
+          Archs.insert(Arch);
+      }
+      if (AMDTriple) {
+        auto TempTC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(
+            *this, *AMDTriple, *HostTC, C.getInputArgs());
+        for (StringRef Arch :
+            getOffloadArchs(C, C.getArgs(), Action::OFK_SYCL, &*TempTC, true))
+          Archs.insert(Arch);
+      }
+      if (!AMDTriple && !NVPTXTriple) {
+        for (StringRef Arch :
+            getOffloadArchs(C, C.getArgs(), Action::OFK_SYCL, nullptr, true))
+          Archs.insert(Arch);
+      }
+      for (StringRef Arch : Archs) {
+        if (NVPTXTriple && IsSYCLSupportedNVidiaGPUArch(StringToOffloadArch(
+                              getProcessorFromTargetID(*NVPTXTriple, Arch)))) {
+          DerivedArchs[NVPTXTriple->getTriple()].insert(Arch);
+        } else if (AMDTriple &&
+                  IsSYCLSupportedAMDGPUArch(StringToOffloadArch(
+                      getProcessorFromTargetID(*AMDTriple, Arch)))) {
+          DerivedArchs[AMDTriple->getTriple()].insert(Arch);
+        } else if (IsSYCLSupportedIntelCPUArch(StringToOffloadArchSYCL(Arch))) {
+          DerivedArchs[getSYCLDeviceTriple("spir64_x86_64").getTriple()].insert(
+              Arch);
+        } else if (IsSYCLSupportedIntelGPUArch(StringToOffloadArchSYCL(Arch))) {
+          StringRef IntelGPUArch;
+          // For Intel Graphics AOT target, valid values for '--offload-arch'
+          // are mapped to valid device names accepted by OCLOC (the Intel GPU AOT
+          // compiler) via the '-device' option. The mapIntelGPUArchName
+          // function maps the accepted values for '--offload-arch' to enable SYCL
+          // offloading to Intel GPUs and the corresponding '-device' value passed
+          // to OCLOC.
+          IntelGPUArch = mapIntelGPUArchName(Arch).data();
+          DerivedArchs[getSYCLDeviceTriple("spir64_gen").getTriple()].insert(
+              IntelGPUArch);
+        } else {
+          Diag(clang::diag::err_drv_invalid_sycl_target) << Arch;
+          return;
+        }
+      }
+      // Emit an error if architecture value is not provided
+      // to --offload-arch.
+      if (Archs.empty()) {
+        Diag(clang::diag::err_drv_sycl_offload_arch_missing_value);
+        return;
+      }
+
+      for (const auto &TripleAndArchs : DerivedArchs)
+        SYCLTriples.insert(TripleAndArchs.first());
+
+      for (const auto &Val : SYCLTriples) {
+        llvm::Triple SYCLTargetTriple(getSYCLDeviceTriple(Val.getKey()));
+        std::string NormalizedName = SYCLTargetTriple.normalize();
+
+        // Make sure we don't have a duplicate triple.
+        auto Duplicate = FoundNormalizedTriples.find(NormalizedName);
+        if (Duplicate != FoundNormalizedTriples.end()) {
+          Diag(clang::diag::warn_drv_sycl_offload_target_duplicate)
+              << Val.getKey() << Duplicate->second;
+          continue;
+        }
+
+        // Store the current triple so that we can check for duplicates in the
+        // following iterations.
+        FoundNormalizedTriples[NormalizedName] = Val.getKey();
+        UniqueSYCLTriplesVec.push_back(SYCLTargetTriple);
+      }
+
+      addSYCLDefaultTriple(C, UniqueSYCLTriplesVec);
+    } else 
+        addSYCLDefaultTriple(C, UniqueSYCLTriplesVec);
 
     // We'll need to use the SYCL and host triples as the key into
-    // getOffloadingDeviceToolChain, because the device toolchains we're
+    // getOffloadToolChain, because the device toolchains we're
     // going to create will depend on both.
     const ToolChain *HostTC = C.getSingleOffloadToolChain<Action::OFK_Host>();
     for (const auto &TT : UniqueSYCLTriplesVec) {
       auto SYCLTC = &getOffloadToolChain(C.getInputArgs(), Action::OFK_SYCL, TT,
                                          HostTC->getTriple());
       C.addOffloadDeviceToolChain(SYCLTC, Action::OFK_SYCL);
+      if (DerivedArchs.contains(TT.getTriple()))
+          KnownArchs[SYCLTC] = DerivedArchs[TT.getTriple()];
     }
   }
 
@@ -6596,7 +6709,7 @@ const ToolChain &Driver::getOffloadToolChain(
       if (Kind == Action::OFK_HIP)
         TC = std::make_unique<toolchains::HIPAMDToolChain>(*this, Target,
                                                            *HostTC, Args);
-      else if (Kind == Action::OFK_OpenMP)
+      else if ((Kind == Action::OFK_OpenMP) || (Kind == Action::OFK_SYCL))
         TC = std::make_unique<toolchains::AMDGPUOpenMPToolChain>(*this, Target,
                                                                  *HostTC, Args);
       break;

clang/lib/Driver/ToolChains/AMDGPUOpenMP.cpp

@@ -46,8 +46,9 @@ void AMDGPUOpenMPToolChain::addClangTargetOptions(
     Action::OffloadKind DeviceOffloadingKind) const {
   HostTC.addClangTargetOptions(DriverArgs, CC1Args, DeviceOffloadingKind);
 
-  assert(DeviceOffloadingKind == Action::OFK_OpenMP &&
-         "Only OpenMP offloading kinds are supported.");
+  assert((DeviceOffloadingKind == Action::OFK_OpenMP ||
+          DeviceOffloadingKind == Action::OFK_SYCL) &&
+         "Only OpenMP or SYCL offloading kinds are supported.");
 
   if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
                           true))

clang/lib/Driver/ToolChains/Clang.cpp

@@ -5091,6 +5091,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   InputInfoList HostOffloadingInputs;
   const InputInfo *CudaDeviceInput = nullptr;
   const InputInfo *OpenMPDeviceInput = nullptr;
+  const InputInfo *SYCLDeviceInput = nullptr;  
   for (const InputInfo &I : Inputs) {
     if (&I == &Input || I.getType() == types::TY_Nothing) {
       // This is the primary input or contains nothing.
@@ -5108,13 +5109,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       CudaDeviceInput = &I;
     } else if (IsOpenMPDevice && !OpenMPDeviceInput) {
       OpenMPDeviceInput = &I;
+    } else if (IsSYCL && !SYCLDeviceInput) {
+      SYCLDeviceInput = &I;
     } else {
       llvm_unreachable("unexpectedly given multiple inputs");
     }
   }
 
   const llvm::Triple *AuxTriple =
-      (IsCuda || IsHIP) ? TC.getAuxTriple() : nullptr;
+      (IsSYCL || IsCuda || IsHIP) ? TC.getAuxTriple() : nullptr;
   bool IsWindowsMSVC = RawTriple.isWindowsMSVCEnvironment();
   bool IsUEFI = RawTriple.isUEFI();
   bool IsIAMCU = RawTriple.isOSIAMCU();
@@ -5208,6 +5211,14 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (IsSYCL) {
     if (IsSYCLDevice) {
+      if (Triple.isNVPTX()) {
+        StringRef GPUArchName = JA.getOffloadingArch();
+        // TODO: Once default arch is moved to at least SM_53, empty arch should
+        // also result in the flag added.
+        if (!GPUArchName.empty() &&
+            StringToOffloadArch(GPUArchName) >= OffloadArch::SM_53)
+          CmdArgs.push_back("-fnative-half-type");
+      }
       // Host triple is needed when doing SYCL device compilations.
       llvm::Triple AuxT = C.getDefaultToolChain().getTriple();
       std::string NormalizedTriple = AuxT.normalize();
@@ -5220,13 +5231,45 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       // Set O2 optimization level by default
       if (!Args.getLastArg(options::OPT_O_Group))
         CmdArgs.push_back("-O2");
+      // Add any predefined macros associated with intel_gpu* type targets
+      // passed in with -fsycl-targets
+      // TODO: Macros are populated during device compilations and saved for
+      // addition to the host compilation. There is no dependence connection
+      // between device and host where we should be able to use the offloading
+      // arch to add the macro to the host compile.
+      auto addTargetMacros = [&](const llvm::Triple &Triple) {
+        if (!Triple.isSPIR() && !Triple.isNVPTX() && !Triple.isAMDGCN())
+          return;
+        SmallString<64> Macro;
+        if ((Triple.isSPIR() &&
+             Triple.getSubArch() == llvm::Triple::SPIRSubArch_gen) ||
+            Triple.isNVPTX() || Triple.isAMDGCN()) {
+          StringRef Device = JA.getOffloadingArch();
+          if (!Device.empty() &&
+              !clang::driver::getGenDeviceMacro(Device).empty()) {
+            Macro = "-D";
+            Macro += clang::driver::getGenDeviceMacro(Device);
+          }
+        } else if (Triple.getSubArch() == llvm::Triple::SPIRSubArch_x86_64)
+          Macro = "-D__SYCL_TARGET_INTEL_X86_64__";
+        if (Macro.size()) {
+          CmdArgs.push_back(Args.MakeArgString(Macro));
+          D.addSYCLTargetMacroArg(Args, Macro);
+        }
+      };
+      addTargetMacros(RawTriple);        
     } else {
       // Add any options that are needed specific to SYCL offload while
       // performing the host side compilation.
 
       // Let the front-end host compilation flow know about SYCL offload
       // compilation.
       CmdArgs.push_back("-fsycl-is-host");
+
+      // Add the SYCL target macro arguments that were generated during the
+      // device compilation step.
+      for (auto &Macro : D.getSYCLTargetMacroArgs())
+        CmdArgs.push_back(Args.MakeArgString(Macro));
     }
 
     // Set options for both host and device.

clang/lib/Driver/ToolChains/Cuda.cpp

@@ -848,8 +848,8 @@ void CudaToolChain::addClangTargetOptions(
 
   StringRef GpuArch = DriverArgs.getLastArgValue(options::OPT_march_EQ);
   assert((DeviceOffloadingKind == Action::OFK_OpenMP ||
-          DeviceOffloadingKind == Action::OFK_Cuda) &&
-         "Only OpenMP or CUDA offloading kinds are supported for NVIDIA GPUs.");
+          DeviceOffloadingKind == Action::OFK_Cuda || DeviceOffloadingKind == Action::OFK_SYCL) &&
+         "Only OpenMP or CUDA or SYCL offloading kinds are supported for NVIDIA GPUs.");
 
   CC1Args.append({"-fcuda-is-device", "-mllvm",
                   "-enable-memcpyopt-without-libcalls",