[Backport to llvm_release_180] initial support for SPV_INTEL_device_barrier (KhronosGroup#3536)

Backport of PR KhronosGroup#3461 into `llvm_release_180`.

All commits applied cleanly.

---------

Co-authored-by: Ben Ashbaugh <ben.ashbaugh@intel.com>

Author: github-actions[bot]

include/LLVMSPIRVExtensions.inc

@@ -57,6 +57,7 @@ EXT(SPV_INTEL_runtime_aligned)
 EXT(SPV_EXT_arithmetic_fence)
 EXT(SPV_INTEL_arithmetic_fence)
 EXT(SPV_INTEL_bfloat16_conversion)
+EXT(SPV_INTEL_device_barrier)
 EXT(SPV_INTEL_joint_matrix)
 EXT(SPV_INTEL_hw_thread_queries)
 EXT(SPV_INTEL_global_variable_decorations)

lib/SPIRV/SPIRVInternal.h

@@ -985,7 +985,7 @@ CallInst *setAttrByCalledFunc(CallInst *Call);
 bool isSPIRVBuiltinVariable(GlobalVariable *GV, SPIRVBuiltinVariableKind *Kind);
 // Transform builtin variable from GlobalVariable to builtin call.
 // e.g.
-// - GlobalInvolcationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x)
+// - GlobalInvocationId[x] -> _Z33__spirv_BuiltInGlobalInvocationIdi(x)
 // - WorkDim -> _Z22__spirv_BuiltInWorkDimv()
 bool lowerBuiltinVariableToCall(GlobalVariable *GV,
                                 SPIRVBuiltinVariableKind Kind);

lib/SPIRV/SPIRVUtil.cpp

@@ -2034,13 +2034,15 @@ static void replaceUsesOfBuiltinVar(Value *V, const APInt &AccumulatedOffset,
     } else if (auto *Load = dyn_cast<LoadInst>(U)) {
       // Figure out which index the accumulated offset corresponds to. If we
       // have a weird offset (e.g., trying to load byte 7), bail out.
-      Type *ScalarTy = ReplacementFunc->getReturnType();
       APInt Index;
-      uint64_t Remainder;
-      APInt::udivrem(AccumulatedOffset, ScalarTy->getScalarSizeInBits() / 8,
-                     Index, Remainder);
-      if (Remainder != 0)
-        llvm_unreachable("Illegal GEP of a SPIR-V builtin variable");
+      Type *ScalarTy = ReplacementFunc->getReturnType();
+      if (!ScalarTy->isIntegerTy(1)) {
+        uint64_t Remainder;
+        APInt::udivrem(AccumulatedOffset, ScalarTy->getScalarSizeInBits() / 8,
+                       Index, Remainder);
+        if (Remainder != 0)
+          llvm_unreachable("Illegal GEP of a SPIR-V builtin variable");
+      }
 
       IRBuilder<> Builder(Load);
       Value *Replacement;

lib/SPIRV/libSPIRV/SPIRVEntry.h

@@ -912,6 +912,8 @@ class SPIRVCapability : public SPIRVEntryNoId<OpCapability> {
       return ExtensionID::SPV_INTEL_function_variants;
     case internal::CapabilityBFloat16ArithmeticINTEL:
       return ExtensionID::SPV_INTEL_bfloat16_arithmetic;
+    case internal::CapabilityDeviceBarrierINTEL:
+      return ExtensionID::SPV_INTEL_device_barrier;
     default:
       return {};
     }

lib/SPIRV/libSPIRV/SPIRVEnum.h

@@ -584,6 +584,8 @@ template <> inline void SPIRVMap<BuiltIn, SPIRVCapVec>::init() {
                {internal::CapabilityHWThreadQueryINTEL});
   ADD_VEC_INIT(internal::BuiltInGlobalHWThreadIDINTEL,
                {internal::CapabilityHWThreadQueryINTEL});
+  ADD_VEC_INIT(internal::BuiltInDeviceBarrierValidINTEL,
+               {internal::CapabilityDeviceBarrierINTEL});
 }
 
 template <> inline void SPIRVMap<MemorySemanticsMask, SPIRVCapVec>::init() {

lib/SPIRV/libSPIRV/SPIRVInstruction.h

@@ -2359,13 +2359,40 @@ class SPIRVControlBarrier : public SPIRVInstruction {
     return getValues(Operands);
   }
 
+  SPIRVCapVec getRequiredCapability() const override {
+    if (isDeviceBarrier()) {
+      return getVec(internal::CapabilityDeviceBarrierINTEL);
+    }
+    return SPIRVInstruction::getRequiredCapability();
+  }
+  std::optional<ExtensionID> getRequiredExtension() const override {
+    if (isDeviceBarrier()) {
+      return ExtensionID::SPV_INTEL_device_barrier;
+    }
+    return std::nullopt;
+  }
+
 protected:
   _SPIRV_DEF_ENCDEC3(ExecScope, MemScope, MemSema)
   void validate() const override {
     assert(OpCode == OC);
     assert(WordCount == 4);
     SPIRVInstruction::validate();
   }
+
+  bool isDeviceBarrier() const {
+    if (!getModule()->isAllowedToUseExtension(
+            ExtensionID::SPV_INTEL_device_barrier))
+      return false;
+    SPIRVValue *ESV = getValue(ExecScope);
+    if (ESV && ESV->getOpCode() == OpConstant) {
+      if (static_cast<SPIRVConstant *>(ESV)->getZExtIntValue() != ScopeDevice) {
+        return false;
+      }
+    }
+    return true;
+  }
+
   SPIRVId ExecScope;
   SPIRVId MemScope = SPIRVID_INVALID;
   SPIRVId MemSema = SPIRVID_INVALID;

lib/SPIRV/libSPIRV/SPIRVIsValidEnum.h

@@ -282,6 +282,7 @@ inline bool isValid(spv::BuiltIn V) {
   case BuiltInCullMaskKHR:
   case internal::BuiltInSubDeviceIDINTEL:
   case internal::BuiltInGlobalHWThreadIDINTEL:
+  case internal::BuiltInDeviceBarrierValidINTEL:
     return true;
   default:
     return false;

lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h

@@ -349,6 +349,8 @@ template <> inline void SPIRVMap<BuiltIn, std::string>::init() {
   add(BuiltInMax, "BuiltInMax");
   add(internal::BuiltInSubDeviceIDINTEL, "BuiltInSubDeviceIDINTEL");
   add(internal::BuiltInGlobalHWThreadIDINTEL, "BuiltInGlobalHWThreadIDINTEL");
+  add(internal::BuiltInDeviceBarrierValidINTEL,
+      "BuiltInDeviceBarrierValidINTEL");
 }
 SPIRV_DEF_NAMEMAP(BuiltIn, SPIRVBuiltInNameMap)
 
@@ -691,6 +693,7 @@ template <> inline void SPIRVMap<Capability, std::string>::init() {
   add(CapabilityFloat8CooperativeMatrixEXT, "Float8CooperativeMatrixEXT");
   add(internal::CapabilityPredicatedIOINTEL, "PredicatedIOINTEL");
   add(internal::CapabilitySigmoidINTEL, "SigmoidINTEL");
+  add(internal::CapabilityDeviceBarrierINTEL, "DeviceBarrierINTEL");
   add(internal::CapabilityFloat4E2M1INTEL, "Float4E2M1INTEL");
   add(internal::CapabilityFloat4E2M1CooperativeMatrixINTEL,
       "Float4E2M1CooperativeMatrixINTEL");

lib/SPIRV/libSPIRV/spirv_internal.hpp

@@ -115,6 +115,7 @@ enum InternalCapability {
   ICapabilityHWThreadQueryINTEL = 6134,
   ICapGlobalVariableDecorationsINTEL = 6146,
   ICapabilitySigmoidINTEL = 6167,
+  ICapabilityDeviceBarrierINTEL = 6185,
   ICapabilityCooperativeMatrixCheckedInstructionsINTEL = 6192,
   ICapabilityFloat4E2M1INTEL = 6212,
   ICapabilityFloat4E2M1CooperativeMatrixINTEL = 6213,
@@ -167,6 +168,7 @@ enum InternalFPEncoding {
 enum InternalBuiltIn {
   IBuiltInSubDeviceIDINTEL = 6135,
   IBuiltInGlobalHWThreadIDINTEL = 6136,
+  IBuiltInDeviceBarrierValidINTEL = 6186,
 };
 
 #define _SPIRV_OP(x, y) constexpr x x##y = static_cast<x>(I##x##y);
@@ -198,6 +200,9 @@ _SPIRV_OP(Op, CooperativeMatrixConstructCheckedINTEL)
 _SPIRV_OP(Capability, CooperativeMatrixInvocationInstructionsINTEL)
 _SPIRV_OP(Op, CooperativeMatrixApplyFunctionINTEL)
 
+_SPIRV_OP(Capability, DeviceBarrierINTEL)
+_SPIRV_OP(BuiltIn, DeviceBarrierValidINTEL)
+
 _SPIRV_OP(Capability, HWThreadQueryINTEL)
 _SPIRV_OP(BuiltIn, SubDeviceIDINTEL)
 _SPIRV_OP(BuiltIn, GlobalHWThreadIDINTEL)

test/extensions/INTEL/SPV_INTEL_device_barrier/device_barrier_runtime_scope_spirv.ll

@@ -0,0 +1,116 @@
+;; kernel void test(global uint* dst)
+;; {
+;;    int scope = magic_get_scope();
+;;    __spirv_ControlBarrier(scope, 1, 264);  // local
+;;    __spirv_ControlBarrier(scope, 1, 520);  // global
+;;    __spirv_ControlBarrier(scope, 1, 2056); // image
+;;
+;;    __spirv_ControlBarrier(scope, 0, 520);  // global, all_svm_devices
+;;    __spirv_ControlBarrier(scope, 1, 520);  // global, device
+;;    __spirv_ControlBarrier(scope, 2, 520);  // global, work_group
+;;    __spirv_ControlBarrier(scope, 3, 520);  // global, subgroup
+;;    __spirv_ControlBarrier(scope, 4, 520);  // global, work_item
+;;}
+
+; Test for SPV_INTEL_device_barrier (SPIR-V friendly LLVM IR)
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-spirv %t.bc -o %t.spv --spirv-ext=+SPV_INTEL_device_barrier
+; RUN: llvm-spirv %t.spv -o %t.spt --to-text
+; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV
+
+; RUN: llvm-spirv %t.spv -o %t.rev.bc -r --spirv-target-env=SPV-IR
+; RUN: llvm-dis %t.rev.bc -o %t.rev.ll
+; RUN: FileCheck < %t.rev.ll %s --check-prefix=CHECK-LLVM
+
+; RUN: llvm-spirv %t.bc -o %t.disabled.spv
+; RUN: llvm-spirv %t.disabled.spv -o %t.disabled.spt --to-text
+; RUN: FileCheck < %t.disabled.spt %s --check-prefix=CHECK-SPIRV-EXTENSION-DISABLED
+
+; ModuleID = 'device_barrier_spirv.cl'
+source_filename = "device_barrier_spirv.cl"
+target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+target triple = "spir64"
+
+; CHECK-SPIRV: Capability DeviceBarrierINTEL
+; CHECK-SPIRV: Extension "SPV_INTEL_device_barrier"
+; CHECK-SPIRV: TypeInt [[UINT:[0-9]+]] 32 0
+;
+;; When the SPV_INTEL_device_barrier extension is not enabled, a runtime variable
+;; should not cause the device barrier extension or capability to be declared.
+; CHECK-SPIRV-EXTENSION-DISABLED-NOT: Capability DeviceBarrierINTEL
+; CHECK-SPIRV-EXTENSION-DISABLED-NOT: Extension "SPV_INTEL_device_barrier"
+;
+; Scopes:
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_CROSS_DEVICE:[0-9]+]] 0 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_DEVICE:[0-9]+]] 1 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_WORK_GROUP:[0-9]+]] 2 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_SUBGROUP:[0-9]+]] 3 {{$}}
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[SCOPE_INVOCATION:[0-9]+]] 4 {{$}}
+;
+; Memory Semantics:
+; 0x8 AcquireRelease + 0x100 WorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQREL_LOCAL:[0-9]+]] 264
+; 0x8 AcquireRelease + 0x200 CrossWorkgroupMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQREL_GLOBAL:[0-9]+]] 520
+; 0x8 AcquireRelease + 0x800 ImageMemory
+; CHECK-SPIRV-DAG: Constant [[UINT]] [[ACQREL_IMAGE:[0-9]+]] 2056
+;
+; Runtime execution scope:
+; CHECK-SPIRV: FunctionCall [[#]] [[EXEC_SCOPE:[0-9]+]] [[#]]
+;
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_LOCAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_IMAGE]]
+;
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_CROSS_DEVICE]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_DEVICE]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_WORK_GROUP]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_SUBGROUP]] [[ACQREL_GLOBAL]]
+; CHECK-SPIRV: ControlBarrier [[EXEC_SCOPE]] [[SCOPE_INVOCATION]] [[ACQREL_GLOBAL]]
+
+; CHECK-LLVM-LABEL: define spir_kernel void @test
+; Function Attrs: convergent norecurse nounwind
+define dso_local spir_kernel void @test(ptr addrspace(1) nocapture noundef readnone align 4 %0) local_unnamed_addr #0 !kernel_arg_addr_space !4 !kernel_arg_access_qual !5 !kernel_arg_type !6 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
+  %2 = call noundef i32 @magic_get_scope()
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 264) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 264) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 2056) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 2056) #1
+
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 0, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 0, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 1, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 1, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 2, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 2, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 3, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 3, i32 520) #1
+  tail call spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef %2, i32 noundef 4, i32 noundef 520) #2
+    ; CHECK-LLVM: call spir_func void @_Z22__spirv_ControlBarrieriii(i32 %2, i32 4, i32 520) #1
+  ret void
+}
+
+; Function Attrs: convergent
+declare dso_local spir_func void @_Z22__spirv_ControlBarrieriii(i32 noundef, i32 noundef, i32 noundef) local_unnamed_addr #1
+
+declare spir_func i32 @magic_get_scope()
+
+attributes #0 = { convergent norecurse nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "uniform-work-group-size"="false" }
+attributes #1 = { convergent "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
+attributes #2 = { convergent nounwind }
+
+!llvm.module.flags = !{!0, !1}
+!opencl.ocl.version = !{!2}
+!opencl.spir.version = !{!2}
+!llvm.ident = !{!3}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"frame-pointer", i32 2}
+!2 = !{i32 2, i32 0}
+!3 = !{!"clang version 15.0.0 (https://github.com/llvm/llvm-project 861386dbd6ff0d91636b7c674c2abb2eccd9d3f2)"}
+!4 = !{i32 1}
+!5 = !{!"none"}
+!6 = !{!"uint*"}
+!7 = !{!""}