Add sat argument to dp4a GenISA builtins

Add sat argument to dp4a internal builtins and GenISA builtins.
Remove add_sat instructions.
Propagate dp4a saturation data trough the compiler.

Author: fda0

IGC/BiFModule/Implementation/IGCBiF_Intrinsics.cl

@@ -806,10 +806,10 @@ void __builtin_IB_media_block_write_ulong2(int image, int2 offset, int width, in
 void __builtin_IB_media_block_write_ulong4(int image, int2 offset, int width, int height, ulong4 pixels);
 void __builtin_IB_media_block_write_ulong8(int image, int2 offset, int width, int height, ulong8 pixels);
 
-int __builtin_IB_dp4a_ss(int c, int a, int b) __attribute__((const));
-int __builtin_IB_dp4a_uu(int c, int a, int b) __attribute__((const));
-int __builtin_IB_dp4a_su(int c, int a, int b) __attribute__((const));
-int __builtin_IB_dp4a_us(int c, int a, int b) __attribute__((const));
+int __builtin_IB_dp4a_ss(int c, int a, int b, bool isSaturated) __attribute__((const));
+int __builtin_IB_dp4a_uu(int c, int a, int b, bool isSaturated) __attribute__((const));
+int __builtin_IB_dp4a_su(int c, int a, int b, bool isSaturated) __attribute__((const));
+int __builtin_IB_dp4a_us(int c, int a, int b, bool isSaturated) __attribute__((const));
 
 #define DECL_SUB_GROUP_OPERATION(type, type_abbr, operation, group_type)  \
 type   __builtin_IB_sub_group_##group_type##_##operation##_##type_abbr(type x) __attribute__((const));

IGC/BiFModule/Languages/OpenCL/IBiF_Dot_Product.cl

@@ -37,14 +37,13 @@ INLINE TYPE_RET OVERLOADABLE dot_acc_sat(TYPE_ARG1 a, TYPE_ARG2 b, TYPE_RET acc)
 #define DEFN_INTEL_DOT_PRODUCT_BUILTIN_SPIRV(TYPE_RET, TYPE_ARG1, TYPE_ARG2, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW, TYPE_SUFFIX_IB)     \
 TYPE_RET SPIRV_OVERLOADABLE SPIRV_BUILTIN(TYPE_SUFFIX##DotKHR, MANGLING_OLD, MANGLING_NEW)(TYPE_ARG1 a, TYPE_ARG2 b)                      \
 {                                                                                                                                         \
-    return __builtin_IB_dp4a_##TYPE_SUFFIX_IB(0, as_int(a), as_int(b));                                                               \
+    return __builtin_IB_dp4a_##TYPE_SUFFIX_IB(0, as_int(a), as_int(b), false);                                                            \
 }
 
-#define DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(TYPE_RET, TYPE_ARG1, TYPE_ARG2, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW, TYPE_SUFFIX_IB, SAT_PREFIX)     \
-TYPE_RET SPIRV_OVERLOADABLE SPIRV_BUILTIN(TYPE_SUFFIX##DotAccSatKHR, MANGLING_OLD, MANGLING_NEW)(TYPE_ARG1 a, TYPE_ARG2 b, TYPE_RET acc)                  \
-{                                                                                                                                                         \
-    TYPE_RET product = __builtin_IB_dp4a_##TYPE_SUFFIX_IB(0, as_int(a), as_int(b));                                                                   \
-    return SPIRV_OCL_BUILTIN(SAT_PREFIX##_add_sat, _i32_i32,)(product, acc);                                                                              \
+#define DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(TYPE_RET, TYPE_ARG1, TYPE_ARG2, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW, TYPE_SUFFIX_IB) \
+TYPE_RET SPIRV_OVERLOADABLE SPIRV_BUILTIN(TYPE_SUFFIX##DotAccSatKHR, MANGLING_OLD, MANGLING_NEW)(TYPE_ARG1 a, TYPE_ARG2 b, TYPE_RET acc)  \
+{                                                                                                                                         \
+    return __builtin_IB_dp4a_##TYPE_SUFFIX_IB(acc, as_int(a), as_int(b), true);                                                           \
 }
 
 #define DEFN_INTEL_DOT_PRODUCT_US(TYPE_RET, TYPE_ARG, MANGLING_OLD, MANGLING_NEW)     \
@@ -62,14 +61,13 @@ INLINE TYPE_RET OVERLOADABLE dot_acc_sat(u##TYPE_ARG a, TYPE_ARG b, TYPE_RET acc
 #define DEFN_INTEL_DOT_PRODUCT_PACKED_BUILTIN_SPIRV(TYPE_RET, TYPE_ARG1, TYPE_ARG2, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW, TYPE_SUFFIX_IB)     \
 TYPE_RET SPIRV_OVERLOADABLE SPIRV_BUILTIN(TYPE_SUFFIX##DotKHR, MANGLING_OLD, MANGLING_NEW)(TYPE_ARG1 a, TYPE_ARG2 b, TYPE_ARG1 packed)           \
 {                                                                                                                                                \
-    return __builtin_IB_dp4a_##TYPE_SUFFIX_IB(0, as_int(a), as_int(b));                                                                      \
+    return __builtin_IB_dp4a_##TYPE_SUFFIX_IB(0, as_int(a), as_int(b), false);                                                                   \
 }
 
-#define DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(TYPE_RET, TYPE_ARG1, TYPE_ARG2, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW, TYPE_SUFFIX_IB, SAT_PREFIX)     \
-TYPE_RET SPIRV_OVERLOADABLE SPIRV_BUILTIN(TYPE_SUFFIX##DotAccSatKHR, MANGLING_OLD, MANGLING_NEW)(TYPE_ARG1 a, TYPE_ARG2 b, TYPE_RET acc, TYPE_ARG1 packed)       \
-{                                                                                                                                                                \
-    TYPE_RET product = __builtin_IB_dp4a_##TYPE_SUFFIX_IB(0, as_int(a), as_int(b));                                                                          \
-    return SPIRV_OCL_BUILTIN(SAT_PREFIX##_add_sat, _i32_i32,)(product, acc);                                                                                     \
+#define DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(TYPE_RET, TYPE_ARG1, TYPE_ARG2, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW, TYPE_SUFFIX_IB)           \
+TYPE_RET SPIRV_OVERLOADABLE SPIRV_BUILTIN(TYPE_SUFFIX##DotAccSatKHR, MANGLING_OLD, MANGLING_NEW)(TYPE_ARG1 a, TYPE_ARG2 b, TYPE_RET acc, TYPE_ARG1 packed) \
+{                                                                                                                                                          \
+    return __builtin_IB_dp4a_##TYPE_SUFFIX_IB(acc, as_int(a), as_int(b), true);                                                                              \
 }
 
 #define DEFN_INTEL_DOT_PRODUCT_PACKED(TYPE_RET, ARG_TYPES, TYPE_SUFFIX, MANGLING_OLD, MANGLING_NEW)     \
@@ -105,12 +103,12 @@ DEFN_INTEL_DOT_PRODUCT_PACKED_BUILTIN_SPIRV(uint, uint, uint, U, _i32_i32_i32, _
 DEFN_INTEL_DOT_PRODUCT_PACKED(uint, uu, U, _i32_i32_i32, _Ruint)
 #endif // __opencl_c_integer_dot_product_input_4x8bit_packed
 #ifdef __opencl_c_integer_dot_product_saturation_accumulation
-DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(uint, uchar4, uchar4, U, _v4i8_v4i8_i32, _Ruint, uu, u)
+DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(uint, uchar4, uchar4, U, _v4i8_v4i8_i32, _Ruint, uu)
 DEFN_INTEL_DOT_PRODUCT_SAT(uint, uchar4, uchar4, U, _v4i8_v4i8_i32, _Ruint)
-DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(uint, ushort2, ushort2, U, _v2i16_v2i16_i32, _Ruint, uu, u)
+DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(uint, ushort2, ushort2, U, _v2i16_v2i16_i32, _Ruint, uu)
 DEFN_INTEL_DOT_PRODUCT_SAT(uint, ushort2, ushort2, U, _v2i16_v2i16_i32, _Ruint)
 #ifdef __opencl_c_integer_dot_product_input_4x8bit_packed
-DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(uint, uint, uint, U, _i32_i32_i32_i32, _Ruint, uu, u)
+DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(uint, uint, uint, U, _i32_i32_i32_i32, _Ruint, uu)
 DEFN_INTEL_DOT_PRODUCT_SAT_PACKED(uint, U, uu, _i32_i32_i32_i32, _Ruint)
 #endif // __opencl_c_integer_dot_product_input_4x8bit_packed
 #endif // __opencl_c_integer_dot_product_saturation_accumulation
@@ -125,12 +123,12 @@ DEFN_INTEL_DOT_PRODUCT_PACKED_BUILTIN_SPIRV(int, int, int, S, _i32_i32_i32, _Rin
 DEFN_INTEL_DOT_PRODUCT_PACKED(int, ss, S, _i32_i32_i32, _Rint)
 #endif // __opencl_c_integer_dot_product_input_4x8bit_packed
 #ifdef __opencl_c_integer_dot_product_saturation_accumulation
-DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, char4, char4, S, _v4i8_v4i8_i32, _Rint, ss, s)
+DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, char4, char4, S, _v4i8_v4i8_i32, _Rint, ss)
 DEFN_INTEL_DOT_PRODUCT_SAT(int, char4, char4, S, _v4i8_v4i8_i32, _Rint)
-DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, short2, short2, S, _v2i16_v2i16_i32, _Rint, ss, s)
+DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, short2, short2, S, _v2i16_v2i16_i32, _Rint, ss)
 DEFN_INTEL_DOT_PRODUCT_SAT(int, short2, short2, S, _v2i16_v2i16_i32, _Rint)
 #ifdef __opencl_c_integer_dot_product_input_4x8bit_packed
-DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(int, int, int, S, _i32_i32_i32_i32, _Rint, ss, s)
+DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(int, int, int, S, _i32_i32_i32_i32, _Rint, ss)
 DEFN_INTEL_DOT_PRODUCT_SAT_PACKED(int, S, ss, _i32_i32_i32_i32, _Rint)
 #endif // __opencl_c_integer_dot_product_input_4x8bit_packed
 #endif // __opencl_c_integer_dot_product_saturation_accumulation
@@ -149,14 +147,14 @@ DEFN_INTEL_DOT_PRODUCT_PACKED(int, su, SU, _i32_i32_i32, _Rint)
 DEFN_INTEL_DOT_PRODUCT_PACKED_US
 #endif // __opencl_c_integer_dot_product_input_4x8bit_packed
 #ifdef __opencl_c_integer_dot_product_saturation_accumulation
-DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, char4, uchar4, SU, _v4i8_v4i8_i32, _Rint, su, s)
+DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, char4, uchar4, SU, _v4i8_v4i8_i32, _Rint, su)
 DEFN_INTEL_DOT_PRODUCT_SAT(int, char4, uchar4, SU, _v4i8_v4i8_i32, _Rint)
 DEFN_INTEL_DOT_PRODUCT_SAT_US(int, char4, _v4i8_v4i8_i32, _Rint)
-DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, short2, ushort2, SU, _v2i16_v2i16_i32, _Rint, su, s)
+DEFN_INTEL_DOT_PRODUCT_SAT_BUILTIN_SPIRV(int, short2, ushort2, SU, _v2i16_v2i16_i32, _Rint, su)
 DEFN_INTEL_DOT_PRODUCT_SAT(int, short2, ushort2, SU, _v2i16_v2i16_i32, _Rint)
 DEFN_INTEL_DOT_PRODUCT_SAT_US(int, short2, _v2i16_v2i16_i32, _Rint)
 #ifdef __opencl_c_integer_dot_product_input_4x8bit_packed
-DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(int, int, uint, SU, _i32_i32_i32_i32, _Rint, su, s)
+DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_BUILTIN_SPIRV(int, int, uint, SU, _i32_i32_i32_i32, _Rint, su)
 DEFN_INTEL_DOT_PRODUCT_SAT_PACKED(int, SU, su, _i32_i32_i32_i32, _Rint)
 DEFN_INTEL_DOT_PRODUCT_SAT_PACKED_US
 #endif // __opencl_c_integer_dot_product_input_4x8bit_packed

IGC/Compiler/CISACodeGen/EmitVISAPass.cpp

@@ -8827,7 +8827,8 @@ void EmitPass::emitAluNoModifier(llvm::GenIntrinsicInst* inst)
 
 void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst* inst)
 {
-    switch (inst->getIntrinsicID())
+    const GenISAIntrinsic::ID intrinsicID = inst->getIntrinsicID();
+    switch (intrinsicID)
     {
     case GenISAIntrinsic::GenISA_OUTPUT:
         emitOutput(inst);
@@ -9415,8 +9416,15 @@ void EmitPass::EmitGenIntrinsicMessage(llvm::GenIntrinsicInst* inst)
     case GenISAIntrinsic::GenISA_dp4a_uu:
     case GenISAIntrinsic::GenISA_dp4a_su:
     case GenISAIntrinsic::GenISA_dp4a_us:
-        emitDP4A(inst);
+    {
+        ConstantInt* constIsSaturated = cast<ConstantInt>(inst->getOperand(3));
+        DstModifier modifier;
+        modifier.sat = constIsSaturated->getValue().getBoolValue();
+
+        bool isAccSigned = intrinsicID != GenISAIntrinsic::GenISA_dp4a_uu;
+        emitDP4A(inst, nullptr, modifier, isAccSigned);
         break;
+    }
     case GenISAIntrinsic::GenISA_evaluateSampler:
         // nothing to do
         break;

IGC/Compiler/CustomSafeOptPass.cpp

@@ -1464,9 +1464,10 @@ void CustomSafeOptPass::matchDp4a(BinaryOperator &I) {
     }
     Value* ValA = Builder.CreateBitCast(VectorA, Builder.getInt32Ty());
     Value* ValB = Builder.CreateBitCast(VectorB, Builder.getInt32Ty());
+    Value* ValSat = Builder.getInt1(false);
 
     Function* Dp4aFun = GenISAIntrinsic::getDeclaration(I.getModule(), IntrinsicID, Builder.getInt32Ty());
-    Value* Res = Builder.CreateCall(Dp4aFun, { AccVal, ValA, ValB });
+    Value* Res = Builder.CreateCall(Dp4aFun, { AccVal, ValA, ValB, ValSat });
     I.replaceAllUsesWith(Res);
 }
 

IGC/Compiler/tests/CustomSafeOptPass/dp4a.ll

@@ -37,7 +37,7 @@ define i32 @test_dp4a_ss_noacc(i32 %src1, i32 %src2) {
 ; CHECK:    [[TMP8:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[H]], i64 3
 ; CHECK:    [[TMP9:%.*]] = bitcast <4 x i8> [[TMP7]] to i32
 ; CHECK:    [[TMP10:%.*]] = bitcast <4 x i8> [[TMP8]] to i32
-; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.ss.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]])
+; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.ss.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]], i1 false)
 ; CHECK:    ret i32 [[TMP11]]
 ;
   %vec1 = bitcast i32 %src1 to <4 x i8>
@@ -90,7 +90,7 @@ define i32 @test_dp4a_ss_end(i32 %src1, i32 %src2, i32 %acc) {
 ; CHECK:    [[TMP8:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[H]], i64 3
 ; CHECK:    [[TMP9:%.*]] = bitcast <4 x i8> [[TMP7]] to i32
 ; CHECK:    [[TMP10:%.*]] = bitcast <4 x i8> [[TMP8]] to i32
-; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.ss.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]])
+; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.ss.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]], i1 false)
 ; CHECK:    [[TMP12:%.*]] = add i32 [[ACC:%.*]], [[TMP11]]
 ; CHECK:    ret i32 [[TMP12]]
 ;
@@ -145,7 +145,7 @@ define i32 @test_dp4a_ss_forw(i32 %src1, i32 %src2, i32 %acc) {
 ; CHECK:    [[TMP8:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[H]], i64 3
 ; CHECK:    [[TMP9:%.*]] = bitcast <4 x i8> [[TMP7]] to i32
 ; CHECK:    [[TMP10:%.*]] = bitcast <4 x i8> [[TMP8]] to i32
-; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.ss.i32(i32 [[ACC:%.*]], i32 [[TMP9]], i32 [[TMP10]])
+; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.ss.i32(i32 [[ACC:%.*]], i32 [[TMP9]], i32 [[TMP10]], i1 false)
 ; CHECK:    ret i32 [[TMP11]]
 ;
   %vec1 = bitcast i32 %src1 to <4 x i8>
@@ -199,7 +199,7 @@ define i32 @test_dp4a_us_noacc(i32 %src1, i32 %src2) {
 ; CHECK:    [[TMP8:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[H]], i64 3
 ; CHECK:    [[TMP9:%.*]] = bitcast <4 x i8> [[TMP7]] to i32
 ; CHECK:    [[TMP10:%.*]] = bitcast <4 x i8> [[TMP8]] to i32
-; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.us.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]])
+; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.us.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]], i1 false)
 ; CHECK:    ret i32 [[TMP11]]
 ;
   %vec1 = bitcast i32 %src1 to <4 x i8>
@@ -253,7 +253,7 @@ define i32 @test_dp4a_su_noacc(i32 %src1, i32 %src2) {
 ; CHECK:    [[TMP8:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[H]], i64 3
 ; CHECK:    [[TMP9:%.*]] = bitcast <4 x i8> [[TMP7]] to i32
 ; CHECK:    [[TMP10:%.*]] = bitcast <4 x i8> [[TMP8]] to i32
-; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.su.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]])
+; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.su.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]], i1 false)
 ; CHECK:    ret i32 [[TMP11]]
 ;
   %vec1 = bitcast i32 %src1 to <4 x i8>
@@ -306,7 +306,7 @@ define i32 @test_dp4a_uu_noacc(i32 %src1, i32 %src2) {
 ; CHECK:    [[TMP8:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[H]], i64 3
 ; CHECK:    [[TMP9:%.*]] = bitcast <4 x i8> [[TMP7]] to i32
 ; CHECK:    [[TMP10:%.*]] = bitcast <4 x i8> [[TMP8]] to i32
-; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.uu.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]])
+; CHECK:    [[TMP11:%.*]] = call i32 @llvm.genx.GenISA.dp4a.uu.i32(i32 0, i32 [[TMP9]], i32 [[TMP10]], i1 false)
 ; CHECK:    ret i32 [[TMP11]]
 ;
   %vec1 = bitcast i32 %src1 to <4 x i8>

IGC/GenISAIntrinsics/generator/input/Intrinsic_definitions.yml

@@ -2872,6 +2872,10 @@ intrinsics:
                 name: Arg2
                 type_definition: *i32
                 comment: "b of char4"
+            - !<ArgumentDefinition>
+                name: Arg3
+                type_definition: *i1
+                comment: "bool isSaturated"
         attributes:
             - !AttributeID "NoUnwind"
         memory_effects:
@@ -2896,6 +2900,10 @@ intrinsics:
                 name: Arg2
                 type_definition: *i32
                 comment: "b of uchar4"
+            - !<ArgumentDefinition>
+                name: Arg3
+                type_definition: *i1
+                comment: "bool isSaturated"
         attributes:
             - !AttributeID "NoUnwind"
         memory_effects:
@@ -2920,6 +2928,10 @@ intrinsics:
                 name: Arg2
                 type_definition: *i32
                 comment: "b of char4"
+            - !<ArgumentDefinition>
+                name: Arg3
+                type_definition: *i1
+                comment: "bool isSaturated"
         attributes:
             - !AttributeID "NoUnwind"
         memory_effects:
@@ -2944,6 +2956,10 @@ intrinsics:
                 name: Arg2
                 type_definition: *i32
                 comment: "b of uchar4"
+            - !<ArgumentDefinition>
+                name: Arg3
+                type_definition: *i1
+                comment: "bool isSaturated"
         attributes:
             - !AttributeID "NoUnwind"
         memory_effects: