diff --git a/include/LLVMSPIRVExtensions.inc b/include/LLVMSPIRVExtensions.inc index 89147228a8..893bee8e66 100644 --- a/include/LLVMSPIRVExtensions.inc +++ b/include/LLVMSPIRVExtensions.inc @@ -78,3 +78,4 @@ EXT(SPV_INTEL_bfloat16_arithmetic) EXT(SPV_INTEL_ternary_bitwise_function) EXT(SPV_INTEL_int4) EXT(SPV_INTEL_function_variants) +EXT(SPV_INTEL_shader_atomic_bfloat16) diff --git a/lib/SPIRV/OCLUtil.cpp b/lib/SPIRV/OCLUtil.cpp index cfd70a27a5..bfcac9befc 100644 --- a/lib/SPIRV/OCLUtil.cpp +++ b/lib/SPIRV/OCLUtil.cpp @@ -647,6 +647,8 @@ template <> void LLVMSPIRVAtomicRmwOpCodeMap::init() { add(llvm::AtomicRMWInst::UMax, OpAtomicUMax); add(llvm::AtomicRMWInst::UMin, OpAtomicUMin); add(llvm::AtomicRMWInst::FAdd, OpAtomicFAddEXT); + add(llvm::AtomicRMWInst::FMin, OpAtomicFMinEXT); + add(llvm::AtomicRMWInst::FMax, OpAtomicFMaxEXT); } } // namespace SPIRV diff --git a/lib/SPIRV/SPIRVWriter.cpp b/lib/SPIRV/SPIRVWriter.cpp index 1cc8d595ba..25e5d33f98 100644 --- a/lib/SPIRV/SPIRVWriter.cpp +++ b/lib/SPIRV/SPIRVWriter.cpp @@ -2282,7 +2282,8 @@ LLVMToSPIRVBase::transValueWithoutDecoration(Value *V, SPIRVBasicBlock *BB, AtomicRMWInst::BinOp Op = ARMW->getOperation(); const bool SupportedAtomicInst = AtomicRMWInst::isFPOperation(Op) - ? (Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub) + ? (Op == AtomicRMWInst::FAdd || Op == AtomicRMWInst::FSub || + Op == AtomicRMWInst::FMin || Op == AtomicRMWInst::FMax) : Op != AtomicRMWInst::Nand; if (!BM->getErrorLog().checkError( SupportedAtomicInst, SPIRVEC_InvalidInstruction, V, diff --git a/lib/SPIRV/libSPIRV/SPIRVInstruction.h b/lib/SPIRV/libSPIRV/SPIRVInstruction.h index a03445fc5c..a1312c0412 100644 --- a/lib/SPIRV/libSPIRV/SPIRVInstruction.h +++ b/lib/SPIRV/libSPIRV/SPIRVInstruction.h @@ -2862,6 +2862,8 @@ class SPIRVAtomicFAddEXTInst : public SPIRVAtomicInstBase { public: llvm::Optional getRequiredExtension() const override { assert(hasType()); + if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) + return ExtensionID::SPV_INTEL_shader_atomic_bfloat16; if (getType()->isTypeFloat(16)) return ExtensionID::SPV_EXT_shader_atomic_float16_add; return ExtensionID::SPV_EXT_shader_atomic_float_add; @@ -2869,6 +2871,8 @@ class SPIRVAtomicFAddEXTInst : public SPIRVAtomicInstBase { SPIRVCapVec getRequiredCapability() const override { assert(hasType()); + if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) + return {internal::CapabilityAtomicBFloat16AddINTEL}; if (getType()->isTypeFloat(16)) return {CapabilityAtomicFloat16AddEXT}; if (getType()->isTypeFloat(32)) @@ -2876,26 +2880,30 @@ class SPIRVAtomicFAddEXTInst : public SPIRVAtomicInstBase { if (getType()->isTypeFloat(64)) return {CapabilityAtomicFloat64AddEXT}; llvm_unreachable( - "AtomicFAddEXT can only be generated for f16, f32, f64 types"); + "AtomicFAddEXT can only be generated for bf16, f16, f32, f64 types"); } }; class SPIRVAtomicFMinMaxEXTBase : public SPIRVAtomicInstBase { public: llvm::Optional getRequiredExtension() const override { + if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) + return ExtensionID::SPV_INTEL_shader_atomic_bfloat16; return ExtensionID::SPV_EXT_shader_atomic_float_min_max; } SPIRVCapVec getRequiredCapability() const override { assert(hasType()); + if (getType()->isTypeFloat(16, FPEncodingBFloat16KHR)) + return {internal::CapabilityAtomicBFloat16MinMaxINTEL}; if (getType()->isTypeFloat(16)) return {CapabilityAtomicFloat16MinMaxEXT}; if (getType()->isTypeFloat(32)) return {CapabilityAtomicFloat32MinMaxEXT}; if (getType()->isTypeFloat(64)) return {CapabilityAtomicFloat64MinMaxEXT}; - llvm_unreachable( - "AtomicF(Min|Max)EXT can only be generated for f16, f32, f64 types"); + llvm_unreachable("AtomicF(Min|Max)EXT can only be generated for bf16, f16, " + "f32, f64 types"); } }; diff --git a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h index 88420e82e6..f1b4c28e19 100644 --- a/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h +++ b/lib/SPIRV/libSPIRV/SPIRVNameMapEnum.h @@ -603,6 +603,9 @@ template <> inline void SPIRVMap::init() { add(CapabilityLongCompositesINTEL, "LongCompositesINTEL"); add(CapabilityOptNoneEXT, "OptNoneEXT"); add(CapabilityAtomicFloat16AddEXT, "AtomicFloat16AddEXT"); + add(internal::CapabilityAtomicBFloat16AddINTEL, "AtomicBFloat16AddINTEL"); + add(internal::CapabilityAtomicBFloat16MinMaxINTEL, + "AtomicBFloat16MinMaxINTEL"); add(CapabilityDebugInfoModuleINTEL, "DebugInfoModuleINTEL"); add(CapabilitySplitBarrierINTEL, "SplitBarrierINTEL"); add(CapabilityGlobalVariableFPGADecorationsINTEL, diff --git a/lib/SPIRV/libSPIRV/spirv_internal.hpp b/lib/SPIRV/libSPIRV/spirv_internal.hpp index 9c4caf84b4..5f7aee371a 100644 --- a/lib/SPIRV/libSPIRV/spirv_internal.hpp +++ b/lib/SPIRV/libSPIRV/spirv_internal.hpp @@ -110,6 +110,8 @@ enum InternalCapability { ICapGlobalVariableDecorationsINTEL = 6146, ICapabilityCooperativeMatrixCheckedInstructionsINTEL = 6192, ICapabilityBFloat16ArithmeticINTEL = 6226, + ICapabilityAtomicBFloat16AddINTEL = 6255, + ICapabilityAtomicBFloat16MinMaxINTEL = 6256, ICapabilityCooperativeMatrixPrefetchINTEL = 6411, ICapabilityComplexFloatMulDivINTEL = 6414, ICapabilityTensorFloat32RoundingINTEL = 6425, @@ -182,6 +184,9 @@ _SPIRV_OP(Capability, BindlessImagesINTEL) _SPIRV_OP(Op, ConvertHandleToImageINTEL) _SPIRV_OP(Op, ConvertHandleToSamplerINTEL) _SPIRV_OP(Op, ConvertHandleToSampledImageINTEL) + +_SPIRV_OP(Capability, AtomicBFloat16AddINTEL) +_SPIRV_OP(Capability, AtomicBFloat16MinMaxINTEL) #undef _SPIRV_OP constexpr SourceLanguage SourceLanguagePython = diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll new file mode 100644 index 0000000000..02c19941a9 --- /dev/null +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_double.ll @@ -0,0 +1,35 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv --spirv-ext=+SPV_EXT_shader_atomic_float_min_max %t.bc -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; CHECK-DAG: Extension "SPV_EXT_shader_atomic_float_min_max" +; CHECK-DAG: Capability AtomicFloat64MinMaxEXT +; CHECK: TypeInt [[Int:[0-9]+]] 32 0 +; CHECK-DAG: Constant [[Int]] [[Scope_Device:[0-9]+]] 1 {{$}} +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK: TypeFloat [[Double:[0-9]+]] 64 +; CHECK: Variable {{[0-9]+}} [[DoublePointer:[0-9]+]] +; CHECK: Constant [[Double]] [[DoubleValue:[0-9]+]] 0 1078263808 + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@f = common dso_local local_unnamed_addr addrspace(1) global double 0.000000e+00, align 8 + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomicrmw_fadd() local_unnamed_addr #0 { +entry: + %0 = atomicrmw fmin double addrspace(1)* @f, double 42.000000e+00 seq_cst +; CHECK: AtomicFMinEXT [[Double]] {{[0-9]+}} [[DoublePointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[DoubleValue]] + %1 = atomicrmw fmax double addrspace(1)* @f, double 42.000000e+00 seq_cst +; CHECK: AtomicFMaxEXT [[Double]] {{[0-9]+}} [[DoublePointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[DoubleValue]] + + ret void +} + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"wchar_size", i32 4} diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll new file mode 100644 index 0000000000..3422253c03 --- /dev/null +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_float.ll @@ -0,0 +1,35 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv --spirv-ext=+SPV_EXT_shader_atomic_float_min_max %t.bc -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; CHECK-DAG: Extension "SPV_EXT_shader_atomic_float_min_max" +; CHECK-DAG: Capability AtomicFloat32MinMaxEXT +; CHECK: TypeInt [[Int:[0-9]+]] 32 0 +; CHECK-DAG: Constant [[Int]] [[Scope_Device:[0-9]+]] 1 {{$}} +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK: TypeFloat [[Float:[0-9]+]] 32 +; CHECK: Variable {{[0-9]+}} [[FPPointer:[0-9]+]] +; CHECK: Constant [[Float]] [[FPValue:[0-9]+]] 1109917696 + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@f = common dso_local local_unnamed_addr addrspace(1) global float 0.000000e+00, align 4 + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomicrmw_fadd() local_unnamed_addr #0 { +entry: + %0 = atomicrmw fmin float addrspace(1)* @f, float 42.000000e+00 seq_cst +; CHECK: AtomicFMinEXT [[Float]] {{[0-9]+}} [[FPPointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[FPValue]] + %1 = atomicrmw fmax float addrspace(1)* @f, float 42.000000e+00 seq_cst +; CHECK: AtomicFMaxEXT [[Float]] {{[0-9]+}} [[FPPointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[FPValue]] + + ret void +} + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"wchar_size", i32 4} diff --git a/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll new file mode 100644 index 0000000000..c653dc2027 --- /dev/null +++ b/test/extensions/EXT/SPV_EXT_shader_atomic_float_min_max/atomicrmw_fminfmax_half.ll @@ -0,0 +1,35 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv --spirv-ext=+SPV_EXT_shader_atomic_float_min_max %t.bc -o %t.spv +; RUN: spirv-val %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; CHECK-DAG: Extension "SPV_EXT_shader_atomic_float_min_max" +; CHECK-DAG: Capability AtomicFloat16MinMaxEXT +; CHECK: TypeInt [[Int:[0-9]+]] 32 0 +; CHECK-DAG: Constant [[Int]] [[Scope_Device:[0-9]+]] 1 {{$}} +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK: TypeFloat [[Half:[0-9]+]] 16 +; CHECK: Variable {{[0-9]+}} [[HalfPointer:[0-9]+]] +; CHECK: Constant [[Half]] [[HalfValue:[0-9]+]] 20800 + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@f = common dso_local local_unnamed_addr addrspace(1) global half 0.000000e+00, align 4 + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomicrmw_fadd() local_unnamed_addr #0 { +entry: + %0 = atomicrmw fmin half addrspace(1)* @f, half 42.000000e+00 seq_cst +; CHECK: AtomicFMinEXT [[Half]] {{[0-9]+}} [[HalfPointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[HalfValue]] + %1 = atomicrmw fmax half addrspace(1)* @f, half 42.000000e+00 seq_cst +; CHECK: AtomicFMaxEXT [[Half]] {{[0-9]+}} [[HalfPointer]] [[Scope_Device]] [[MemSem_SequentiallyConsistent]] [[HalfValue]] + + ret void +} + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"wchar_size", i32 4} diff --git a/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFAddEXT.ll b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFAddEXT.ll new file mode 100644 index 0000000000..c0209e8279 --- /dev/null +++ b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFAddEXT.ll @@ -0,0 +1,30 @@ +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_shader_atomic_bfloat16,+SPV_KHR_bfloat16 -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv --spirv-target-env=SPV-IR -r %t.spv -o %t.rev.bc +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefixes=CHECK-LLVM-SPV + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +; CHECK-SPIRV-DAG: Capability AtomicBFloat16AddINTEL +; CHECK-SPIRV-DAG: Capability BFloat16TypeKHR +; CHECK-SPIRV-DAG: Extension "SPV_INTEL_shader_atomic_bfloat16" +; CHECK-SPIRV-DAG: Extension "SPV_KHR_bfloat16" + +; CHECK-SPIRV: TypeFloat [[BFLOAT:[0-9]+]] 16 0 + +; Function Attrs: convergent norecurse nounwind +define dso_local spir_func bfloat @test_AtomicFAddEXT_bfloat(ptr addrspace(4) align 2 dereferenceable(4) %Arg) { +entry: + %0 = addrspacecast ptr addrspace(4) %Arg to ptr addrspace(1) + ; CHECK-SPIRV: AtomicFAddEXT [[BFLOAT]] + ; CHECK-LLVM-SPV: call spir_func bfloat @_Z21__spirv_AtomicFAddEXTPU3AS1u6__bf16iiu6__bf16({{.*}}bfloat + %ret = tail call spir_func bfloat @_Z21__spirv_AtomicFAddEXTPU3AS1u6__bf16iiu6__bf16(ptr addrspace(1) %0, i32 1, i32 896, bfloat 1.000000e+00) + ret bfloat %ret +} + +; Function Attrs: convergent +declare dso_local spir_func bfloat @_Z21__spirv_AtomicFAddEXTPU3AS1u6__bf16iiu6__bf16(ptr addrspace(1), i32, i32, bfloat) diff --git a/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFMaxEXT.ll b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFMaxEXT.ll new file mode 100644 index 0000000000..839d9b6520 --- /dev/null +++ b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFMaxEXT.ll @@ -0,0 +1,30 @@ +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_shader_atomic_bfloat16,+SPV_KHR_bfloat16 -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv --spirv-target-env=SPV-IR -r %t.spv -o %t.rev.bc +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefixes=CHECK-LLVM-SPV + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +; CHECK-SPIRV-DAG: Capability AtomicBFloat16MinMaxINTEL +; CHECK-SPIRV-DAG: Capability BFloat16TypeKHR +; CHECK-SPIRV-DAG: Extension "SPV_INTEL_shader_atomic_bfloat16" +; CHECK-SPIRV-DAG: Extension "SPV_KHR_bfloat16" + +; CHECK-SPIRV: TypeFloat [[BFLOAT:[0-9]+]] 16 0 + +; Function Attrs: convergent norecurse nounwind +define dso_local spir_func bfloat @test_AtomicFMaxEXT_bfloat(ptr addrspace(4) align 2 dereferenceable(4) %Arg) { +entry: + %0 = addrspacecast ptr addrspace(4) %Arg to ptr addrspace(1) + ; CHECK-SPIRV: AtomicFMaxEXT [[BFLOAT]] + ; CHECK-LLVM-SPV: call spir_func bfloat @_Z21__spirv_AtomicFMaxEXTPU3AS1u6__bf16iiu6__bf16({{.*}}bfloat + %ret = tail call spir_func bfloat @_Z21__spirv_AtomicFMaxEXTPU3AS1u6__bf16iiu6__bf16(ptr addrspace(1) %0, i32 1, i32 896, bfloat 1.000000e+00) + ret bfloat %ret +} + +; Function Attrs: convergent +declare dso_local spir_func bfloat @_Z21__spirv_AtomicFMaxEXTPU3AS1u6__bf16iiu6__bf16(ptr addrspace(1), i32, i32, bfloat) diff --git a/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFMinEXT.ll b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFMinEXT.ll new file mode 100644 index 0000000000..8c2bdfa221 --- /dev/null +++ b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/AtomicFMinEXT.ll @@ -0,0 +1,30 @@ +; RUN: llvm-as %s -o %t.bc +; RUN: llvm-spirv %t.bc --spirv-ext=+SPV_INTEL_shader_atomic_bfloat16,+SPV_KHR_bfloat16 -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o %t.spt +; RUN: FileCheck < %t.spt %s --check-prefix=CHECK-SPIRV + +; RUN: llvm-spirv --spirv-target-env=SPV-IR -r %t.spv -o %t.rev.bc +; RUN: llvm-dis %t.rev.bc -o - | FileCheck %s --check-prefixes=CHECK-LLVM-SPV + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-n8:16:32:64" +target triple = "spir64-unknown-unknown" + +; CHECK-SPIRV-DAG: Capability AtomicBFloat16MinMaxINTEL +; CHECK-SPIRV-DAG: Capability BFloat16TypeKHR +; CHECK-SPIRV-DAG: Extension "SPV_INTEL_shader_atomic_bfloat16" +; CHECK-SPIRV-DAG: Extension "SPV_KHR_bfloat16" + +; CHECK-SPIRV: TypeFloat [[BFLOAT:[0-9]+]] 16 0 + +; Function Attrs: convergent norecurse nounwind +define dso_local spir_func bfloat @test_AtomicFMinEXT_bfloat(ptr addrspace(4) align 2 dereferenceable(4) %Arg) { +entry: + %0 = addrspacecast ptr addrspace(4) %Arg to ptr addrspace(1) + ; CHECK-SPIRV: AtomicFMinEXT [[BFLOAT]] + ; CHECK-LLVM-SPV: call spir_func bfloat @_Z21__spirv_AtomicFMinEXTPU3AS1u6__bf16iiu6__bf16({{.*}}bfloat + %ret = tail call spir_func bfloat @_Z21__spirv_AtomicFMinEXTPU3AS1u6__bf16iiu6__bf16(ptr addrspace(1) %0, i32 1, i32 896, bfloat 1.000000e+00) + ret bfloat %ret +} + +; Function Attrs: convergent +declare dso_local spir_func bfloat @_Z21__spirv_AtomicFMinEXTPU3AS1u6__bf16iiu6__bf16(ptr addrspace(1), i32, i32, bfloat) diff --git a/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/atomicrmw_fadd.ll b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/atomicrmw_fadd.ll new file mode 100644 index 0000000000..b770315150 --- /dev/null +++ b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/atomicrmw_fadd.ll @@ -0,0 +1,34 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_shader_atomic_bfloat16,+SPV_KHR_bfloat16 %t.bc -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; CHECK-DAG: Extension "SPV_INTEL_shader_atomic_bfloat16" +; CHECK-DAG: Extension "SPV_KHR_bfloat16" +; CHECK-DAG: Capability AtomicBFloat16AddINTEL +; CHECK-DAG: Capability BFloat16TypeKHR +; CHECK: TypeInt [[Int:[0-9]+]] 32 0 +; CHECK-DAG: Constant [[Int]] [[Device:[0-9]+]] 1 {{$}} +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK: TypeFloat [[BFloat:[0-9]+]] 16 0 +; CHECK: Variable {{[0-9]+}} [[BFloatPointer:[0-9]+]] +; CHECK: Constant [[BFloat]] [[BFloatValue:[0-9]+]] 16936 + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@f = common dso_local local_unnamed_addr addrspace(1) global bfloat 0.000000e+00, align 8 + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomicrmw_fadd() local_unnamed_addr #0 { +entry: + %0 = atomicrmw fadd ptr addrspace(1) @f, bfloat 42.000000e+00 seq_cst +; CHECK: AtomicFAddEXT [[BFloat]] {{[0-9]+}} [[BFloatPointer]] [[Device]] [[MemSem_SequentiallyConsistent]] [[BFloatValue]] + + ret void +} + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"wchar_size", i32 4} diff --git a/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/atomicrmw_fminfmax.ll b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/atomicrmw_fminfmax.ll new file mode 100644 index 0000000000..303bbf9aa5 --- /dev/null +++ b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/atomicrmw_fminfmax.ll @@ -0,0 +1,36 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_shader_atomic_bfloat16,+SPV_KHR_bfloat16 %t.bc -o %t.spv +; RUN: llvm-spirv -to-text %t.spv -o - | FileCheck %s + +; CHECK-DAG: Extension "SPV_INTEL_shader_atomic_bfloat16" +; CHECK-DAG: Extension "SPV_KHR_bfloat16" +; CHECK-DAG: AtomicBFloat16MinMaxINTEL +; CHECK-DAG: Capability BFloat16TypeKHR +; CHECK: TypeInt [[Int:[0-9]+]] 32 0 +; CHECK-DAG: Constant [[Int]] [[Device:[0-9]+]] 1 {{$}} +; CHECK-DAG: Constant [[Int]] [[MemSem_SequentiallyConsistent:[0-9]+]] 16 +; CHECK: TypeFloat [[BFloat:[0-9]+]] 16 0 +; CHECK: Variable {{[0-9]+}} [[BFloatPointer:[0-9]+]] +; CHECK: Constant [[BFloat]] [[BFloatValue:[0-9]+]] 16936 + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@f = common dso_local local_unnamed_addr addrspace(1) global bfloat 0.000000e+00, align 4 + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomicrmw_fadd() local_unnamed_addr #0 { +entry: + %0 = atomicrmw fmin ptr addrspace(1) @f, bfloat 42.000000e+00 seq_cst +; CHECK: AtomicFMinEXT [[BFloat]] {{[0-9]+}} [[BFloatPointer]] [[Device]] [[MemSem_SequentiallyConsistent]] [[BFloatValue]] + %1 = atomicrmw fmax ptr addrspace(1) @f, bfloat 42.000000e+00 seq_cst +; CHECK: AtomicFMaxEXT [[BFloat]] {{[0-9]+}} [[BFloatPointer]] [[Device]] [[MemSem_SequentiallyConsistent]] [[BFloatValue]] + + ret void +} + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"wchar_size", i32 4} diff --git a/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/negative.ll b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/negative.ll new file mode 100644 index 0000000000..6d01cce8b1 --- /dev/null +++ b/test/extensions/INTEL/SPV_INTEL_shader_atomic_bfloat16/negative.ll @@ -0,0 +1,29 @@ +; RUN: llvm-as < %s -o %t.bc +; RUN: not llvm-spirv --spirv-ext=+SPV_INTEL_shader_atomic_bfloat16 %t.bc 2>&1 | FileCheck %s --check-prefix=CHECK-NO-BF +; RUN: not llvm-spirv --spirv-ext=+SPV_KHR_bfloat16 %t.bc 2>&1 | FileCheck %s --check-prefix=CHECK-NO-ATOM + +; CHECK-NO-BF: RequiresExtension: Feature requires the following SPIR-V extension: +; CHECK-NO-BF-NEXT: SPV_KHR_bfloat16 +; CHECK-NO-BF-NEXT: NOTE: LLVM module contains bfloat type, translation of which requires this extension + +; CHECK-NO-ATOM: RequiresExtension: Feature requires the following SPIR-V extension: +; CHECK-NO-ATOM-NEXT: SPV_INTEL_shader_atomic_bfloat16 + +target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" +target triple = "spir64" + +@f = common dso_local local_unnamed_addr addrspace(1) global bfloat 0.000000e+00, align 8 + +; Function Attrs: nounwind +define dso_local spir_func void @test_atomicrmw_fadd() local_unnamed_addr #0 { +entry: + %0 = atomicrmw fadd ptr addrspace(1) @f, bfloat 42.000000e+00 seq_cst + + ret void +} + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"wchar_size", i32 4}