[Backport to llvm_release_140] initial support for SPV_INTEL_device_barrier (#3556)

Author: bashbaug

include/LLVMSPIRVExtensions.inc

@@ -56,6 +56,7 @@ EXT(SPV_INTEL_debug_module)
 EXT(SPV_INTEL_runtime_aligned)
 EXT(SPV_INTEL_arithmetic_fence)
 EXT(SPV_INTEL_bfloat16_conversion)
+EXT(SPV_INTEL_device_barrier)
 EXT(SPV_INTEL_joint_matrix)
 EXT(SPV_INTEL_hw_thread_queries)
 EXT(SPV_INTEL_global_variable_decorations)

lib/SPIRV/SPIRVInternal.h

@@ -1110,7 +1110,7 @@ CallInst *setAttrByCalledFunc(CallInst *Call);
 bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind);
 // Transform builtin variable from GlobalVariable to builtin call.
 // e.g.
-// - GlobalInvolcationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x)
+// - GlobalInvocationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x)
 // - WorkDim -> _Z22__spirv_BuiltInWorkDimv()
 bool lowerBuiltinVariableToCall(GlobalVariable *GV,
                                 SPIRVBuiltinVariableKind Kind);

lib/SPIRV/SPIRVUtil.cpp

@@ -2050,13 +2050,15 @@ static void replaceUsesOfBuiltinVar(Value *V, const APInt &AccumulatedOffset,
     } else if (auto *Load = dyn_cast<LoadInst>(U)) {
       // Figure out which index the accumulated offset corresponds to. If we
       // have a weird offset (e.g., trying to load byte 7), bail out.
-      Type *ScalarTy = ReplacementFunc->getReturnType();
       APInt Index;
-      uint64_t Remainder;
-      APInt::udivrem(AccumulatedOffset, ScalarTy->getScalarSizeInBits() / 8,
-                     Index, Remainder);
-      if (Remainder != 0)
-        llvm_unreachable("Illegal GEP of a SPIR-V builtin variable");
+      Type *ScalarTy = ReplacementFunc->getReturnType();
+      if (!ScalarTy->isIntegerTy(1)) {
+        uint64_t Remainder;
+        APInt::udivrem(AccumulatedOffset, ScalarTy->getScalarSizeInBits() / 8,
+                       Index, Remainder);
+        if (Remainder != 0)
+          llvm_unreachable("Illegal GEP of a SPIR-V builtin variable");
+      }
 
       IRBuilder<> Builder(Load);
       Value *Replacement;

lib/SPIRV/libSPIRV/SPIRVEntry.h

@@ -906,6 +906,8 @@ class SPIRVCapability : public SPIRVEntryNoId<OpCapability> {
       return ExtensionID::SPV_INTEL_vector_compute;
     case internal::CapabilityBFloat16ArithmeticINTEL:
       return ExtensionID::SPV_INTEL_bfloat16_arithmetic;
+    case internal::CapabilityDeviceBarrierINTEL:
+      return ExtensionID::SPV_INTEL_device_barrier;
     default:
       return {};
     }

lib/SPIRV/libSPIRV/SPIRVEnum.h

@@ -547,6 +547,8 @@ template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {
                {internal::CapabilityHWThreadQueryINTEL});
   ADD_VEC_INIT(internal::BuiltInGlobalHWThreadIDINTEL,
                {internal::CapabilityHWThreadQueryINTEL});
+  ADD_VEC_INIT(internal::BuiltInDeviceBarrierValidINTEL,
+               {internal::CapabilityDeviceBarrierINTEL});
 }
 
 template <> inline void SPIRVMap<MemorySemanticsMask, SPIRVCapVec>::init() {

lib/SPIRV/libSPIRV/SPIRVInstruction.h

@@ -2362,13 +2362,40 @@ class SPIRVControlBarrier : public SPIRVInstruction {
     return getValues(Operands);
   }
 
+  SPIRVCapVec getRequiredCapability() const override {
+    if (isDeviceBarrier()) {
+      return getVec(internal::CapabilityDeviceBarrierINTEL);
+    }
+    return SPIRVInstruction::getRequiredCapability();
+  }
+  llvm::Optional<ExtensionID> getRequiredExtension() const override {
+    if (isDeviceBarrier()) {
+      return ExtensionID::SPV_INTEL_device_barrier;
+    }
+    return {};
+  }
+
 protected:
   _SPIRV_DEF_ENCDEC3(ExecScope, MemScope, MemSema)
   void validate() const override {
     assert(OpCode == OC);
     assert(WordCount == 4);
     SPIRVInstruction::validate();
   }
+
+  bool isDeviceBarrier() const {
+    if (!getModule()->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_device_barrier))
+      return false;
+    SPIRVValue *ESV = getValue(ExecScope);
+    if (ESV && ESV->getOpCode() == OpConstant) {
+      if (static_cast<SPIRVConstant *>(ESV)->getZExtIntValue() != ScopeDevice) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   SPIRVId ExecScope;
   SPIRVId MemScope;
   SPIRVId MemSema;

lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h

@@ -282,6 +282,7 @@ inline bool isValid(spv::BuiltIn V) {
   case BuiltInCullMaskKHR:
   case internal::BuiltInSubDeviceIDINTEL:
   case internal::BuiltInGlobalHWThreadIDINTEL:
+  case internal::BuiltInDeviceBarrierValidINTEL:
     return true;
   default:
     return false;

lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h

@@ -328,6 +328,8 @@ template <> inline void SPIRVMap<BuiltIn, std::string>::init() {
   add(BuiltInMax, "BuiltInMax");
   add(internal::BuiltInSubDeviceIDINTEL, "BuiltInSubDeviceIDINTEL");
   add(internal::BuiltInGlobalHWThreadIDINTEL, "BuiltInGlobalHWThreadIDINTEL");
+  add(internal::BuiltInDeviceBarrierValidINTEL,
+      "BuiltInDeviceBarrierValidINTEL");
 }
 SPIRV_DEF_NAMEMAP(BuiltIn, SPIRVBuiltInNameMap)
 
@@ -655,6 +657,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilityFloat8CooperativeMatrixEXT, "Float8CooperativeMatrixEXT");
   add(internal::CapabilityPredicatedIOINTEL, "PredicatedIOINTEL");
   add(internal::CapabilitySigmoidINTEL, "SigmoidINTEL");
+  add(internal::CapabilityDeviceBarrierINTEL, "DeviceBarrierINTEL");
   add(internal::CapabilityAtomicBFloat16AddINTEL, "AtomicBFloat16AddINTEL");
   add(internal::CapabilityAtomicBFloat16MinMaxINTEL,
       "AtomicBFloat16MinMaxINTEL");

lib/SPIRV/libSPIRV/spirv_internal.hpp

@@ -121,6 +121,7 @@ enum InternalCapability {
   ICapFPArithmeticFenceINTEL = 6144,
   ICapGlobalVariableDecorationsINTEL = 6146,
   ICapabilitySigmoidINTEL = 6167,
+  ICapabilityDeviceBarrierINTEL = 6185,
   ICapabilityCooperativeMatrixCheckedInstructionsINTEL = 6192,
   ICapabilityFloat4E2M1INTEL = 6212,
   ICapabilityFloat4E2M1CooperativeMatrixINTEL = 6213,
@@ -174,6 +175,7 @@ enum InternalFPEncoding {
 enum InternalBuiltIn {
   IBuiltInSubDeviceIDINTEL = 6135,
   IBuiltInGlobalHWThreadIDINTEL = 6136,
+  IBuiltInDeviceBarrierValidINTEL = 6186,
 };
 
 enum class LoadCacheControlINTEL {
@@ -216,6 +218,9 @@ _SPIRV_OP(Op, CooperativeMatrixLoadCheckedINTEL)
 _SPIRV_OP(Op, CooperativeMatrixStoreCheckedINTEL)
 _SPIRV_OP(Op, CooperativeMatrixConstructCheckedINTEL)
 
+_SPIRV_OP(Capability, DeviceBarrierINTEL)
+_SPIRV_OP(BuiltIn, DeviceBarrierValidINTEL)
+
 _SPIRV_OP(Capability, HWThreadQueryINTEL)
 _SPIRV_OP(BuiltIn, SubDeviceIDINTEL)
 _SPIRV_OP(BuiltIn, GlobalHWThreadIDINTEL)

test/extensions/INTEL/SPV_INTEL_device_barrier/device_barrier_runtime_scope_spirv.ll

@@ -0,0 +1,116 @@
+;; kernel void test(global uint* dst)
+;; {
+;;    int scope = magic_get_scope();
+;;    __spirv_ControlBarrier(scope, 1, 264);  // local
+;;    __spirv_ControlBarrier(scope, 1, 520);  // global
+;;    __spirv_ControlBarrier(scope, 1, 2056); // image
+;;
+;;    __spirv_ControlBarrier(scope, 0, 520);  // global, all_svm_devices
+;;    __spirv_ControlBarrier(scope, 1, 520);  // global, device
+;;    __spirv_ControlBarrier(scope, 2, 520);  // global, work_group
+;;    __spirv_ControlBarrier(scope, 3, 520);  // global, subgroup
+;;    __spirv_ControlBarrier(scope, 4, 520);  // global, work_item
+;;}
+
+; Test for SPV_INTEL_device_barrier (SPIR-V friendly LLVM IR)
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+SPV_INTEL_device_barrier
+; RUN: llvm-spirv %t.spv -o %t.spt --to-text
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+
+; RUN: llvm-spirv %t.spv -o %t.rev.bc -r --spirv-target-env=SPV-IR
+; RUN: llvm-dis %t.rev.bc -o %t.rev.ll
+; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM
+
+; RUN: llvm-spirv %t.bc -o %t.disabled.spv
+; RUN: llvm-spirv %t.disabled.spv -o %t.disabled.spt --to-text
+; RUN: FileCheck < %t.disabled.spt %s --check-prefix=CHECK-SPIRV-EXTENSION-DISABLED
+
+; ModuleID = 'device_barrier_spirv.cl'
+source_filename = "device_barrier_spirv.cl"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+; CHECK-SPIRV: Capability DeviceBarrierINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_device_barrier"
+; CHECK-SPIRV: TypeInt [[UINT:[0-9]+]] 32 0
+;
+;; When the SPV_INTEL_device_barrier extension is not enabled, a runtime variable
+;; should not cause the device barrier extension or capability to be declared.
+; CHECK-SPIRV-EXTENSION-DISABLED-NOT: Capability DeviceBarrierINTEL
+; CHECK-SPIRV-EXTENSION-DISABLED-NOT: Extension "SPV_INTEL_device_barrier"
+;
+; Scopes:
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_CROSS_DEVICE:[0-9]+]] 0 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_DEVICE:[0-9]+]] 1 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_WORK_GROUP:[0-9]+]] 2 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_SUBGROUP:[0-9]+]] 3 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_INVOCATION:[0-9]+]] 4 {{$}}
+;
+; Memory Semantics:
+; 0x8 AcquireRelease + 0x100 WorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQREL_LOCAL:[0-9]+]] 264
+; 0x8 AcquireRelease + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQREL_GLOBAL:[0-9]+]] 520
+; 0x8 AcquireRelease + 0x800 ImageMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQREL_IMAGE:[0-9]+]] 2056
+;
+; Runtime execution scope:
+; CHECK-SPIRV: FunctionCall [[#]] [[EXEC_SCOPE:[0-9]+]] [[#]]
+;
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_LOCAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_IMAGE]]
+;
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_CROSS_DEVICE]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_WORK_GROUP]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_SUBGROUP]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_INVOCATION]] [[ACQREL_GLOBAL]]
+
+; CHECK-LLVM-LABEL: define spir_kernel void @test
+; Function Attrs: convergent norecurse nounwind
+define dso_local spir_kernel void @test(i32 addrspace(1)* nocapture noundef readnone %0) local_unnamed_addr #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
+  %2 = call noundef i32 @magic_get_scope()
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 264) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 264) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 2056) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 2056) #1
+
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 0, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 0, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 2, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 2, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 3, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 3, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 4, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 4, i32 520) #1
+  ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
+
+declare spir_func i32 @magic_get_scope()
+
+attributes #0 = { convergent norecurse nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nounwind }
+
+!llvm.module.flags = !{!0, !1}
+!opencl.ocl.version = !{!2}
+!opencl.spir.version = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"frame-pointer", i32 2}
+!2 = !{i32 2, i32 0}
+!3 = !{!"clang version 15.0.0 (https://github.com/llvm/llvm-project 861386dbd6ff0d91636b7c674c2abb2eccd9d3f2)"}
+!4 = !{i32 1}
+!5 = !{!"none"}
+!6 = !{!"uint*"}
+!7 = !{!""}