finish hooking up CLMUL to selectiondag?

Author: oscardssmith

llvm/docs/LangRef.rst

@@ -18190,7 +18190,7 @@ Semantics:
 """"""""""
 
 The ‘llvm.clmul’ intrinsic computes carryless multiply of ``%a`` and ``%b``, which is the result
-of applying the standard multiplication algorithm if you replace all of the aditions with exclusive ors.
+of applying the standard multiplication algorithm if you replace all of the additions with exclusive ors.
 The vector intrinsics, such as llvm.clmul.v4i32, operate on a per-element basis and the element order is not affected.
 
 Examples

llvm/include/llvm/IR/Intrinsics.td

@@ -1428,7 +1428,7 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in {
   def int_fshr : DefaultAttrsIntrinsic<[llvm_anyint_ty],
       [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
   def int_clmul : DefaultAttrsIntrinsic<[llvm_anyint_ty],
-      [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>;
+      [LLVMMatchType<0>, LLVMMatchType<0>]>;
 }
 
 let IntrProperties = [IntrNoMem, IntrSpeculatable,

llvm/lib/CodeGen/IntrinsicLowering.cpp

@@ -208,7 +208,7 @@ static Value *LowerCLMUL(LLVMContext &Context, Value *V1, Value *V2, Instruction
   Value *Res = ConstantInt::get(V1->getType(), 0);
   Value *Zero = ConstantInt::get(V1->getType(), 0);
   Value *One = ConstantInt::get(V1->getType(), 1);
-  for (unsigned I = 1; I < BitSize; I ++) {
+  for (unsigned I = 1; I < BitSize; I++) {
     Value *LowBit = Builder.CreateAnd(V1, One, "clmul.isodd");
     Value *Pred = Builder.CreateSelect(LowBit, V2, Zero, "clmul.V2_or_zero");
     Res = Builder.CreateXor(Res, Pred, "clmul.Res");

llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

@@ -209,7 +209,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::VP_XOR:
   case ISD::VP_ADD:
   case ISD::VP_SUB:
-  case ISD::VP_MUL:      Res = PromoteIntRes_SimpleIntBinOp(N); break;
+  case ISD::VP_MUL:
+  case ISD::CLMUL:      Res = PromoteIntRes_SimpleIntBinOp(N); break;
 
   case ISD::ABDS:
   case ISD::AVGCEILS:
@@ -3140,6 +3141,10 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
     ExpandIntRes_FunnelShift(N, Lo, Hi);
     break;
 
+  case ISD::CLMUL:
+    ExpandIntRes_CLMUL(N, Lo, Hi);
+    break;
+
   case ISD::VSCALE:
     ExpandIntRes_VSCALE(N, Lo, Hi);
     break;
@@ -5476,6 +5481,35 @@ void DAGTypeLegalizer::ExpandIntRes_FunnelShift(SDNode *N, SDValue &Lo,
   Hi = DAG.getNode(Opc, DL, HalfVT, Select3, Select2, NewShAmt);
 }
 
+void DAGTypeLegalizer::ExpandIntRes_CLMUL(SDNode *N, SDValue &Lo,
+                                                SDValue &Hi) {
+  // Values numbered from least significant to most significant.
+  SDValue In1, In2, In3, In4;
+  GetExpandedInteger(N->getOperand(0), In3, In4);
+  GetExpandedInteger(N->getOperand(1), In1, In2);
+  EVT HalfVT = In1.getValueType();
+  SDLoc DL(N);
+  
+  // CLMUL is carryless so Lo is computed from the low half
+  Lo = DAG.getNode(ISD::CLMUL, DL, HalfVT, In1, In3);
+  // the high bits not included in CLMUL(A,B) can be computed by
+  // BITREVERSE(CLMUL(BITREVERSE(A), BITREVERSE(B))) >> 1
+  // Therefore we can compute the 2 hi/lo cross products
+  // and the the overflow of the low product
+  // and xor them together to compute HI
+  SDValue BitRevIn1 = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, In1);
+  SDValue BitRevIn3 = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, In3);
+  SDValue BitRevLoHi = DAG.getNode(ISD::CLMUL, DL, HalfVT, BitRevIn1, BitRevIn3);
+  SDValue LoHi = DAG.getNode(ISD::BITREVERSE, DL, HalfVT, BitRevLoHi);
+  SDValue One = DAG.getConstant(0, DL, HalfVT);
+  Hi = DAG.getNode(ISD::SRL, DL, HalfVT, LoHi, One);
+  
+  SDValue HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, In1, In4);
+  Hi = DAG.getNode(ISD::XOR, DL, HalfVT, Hi, HITMP);
+  HITMP = DAG.getNode(ISD::CLMUL, DL, HalfVT, In2, In3);
+  Hi = DAG.getNode(ISD::XOR, DL, HalfVT, Hi, HITMP);
+}
+
 void DAGTypeLegalizer::ExpandIntRes_VSCALE(SDNode *N, SDValue &Lo,
                                            SDValue &Hi) {
   EVT VT = N->getValueType(0);

llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h

@@ -511,6 +511,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
 
   void ExpandIntRes_Rotate            (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_FunnelShift       (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandIntRes_CLMUL             (SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void ExpandIntRes_VSCALE            (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_READ_REGISTER(SDNode *N, SDValue &Lo, SDValue &Hi);

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

@@ -166,6 +166,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SMAX:
   case ISD::UMIN:
   case ISD::UMAX:
+  case ISD::CLMUL:
 
   case ISD::SADDSAT:
   case ISD::UADDSAT:
@@ -1330,6 +1331,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SMAX: case ISD::VP_SMAX:
   case ISD::UMIN: case ISD::VP_UMIN:
   case ISD::UMAX: case ISD::VP_UMAX:
+  case ISD::CLMUL:
   case ISD::SADDSAT: case ISD::VP_SADDSAT:
   case ISD::UADDSAT: case ISD::VP_UADDSAT:
   case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
@@ -4764,6 +4766,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::SSUBSAT: case ISD::VP_SSUBSAT:
   case ISD::SSHLSAT:
   case ISD::USHLSAT:
+  case ISD::CLMUL:
   case ISD::ROTL:
   case ISD::ROTR:
   case ISD::AVGFLOORS:

llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp

@@ -8293,26 +8293,22 @@ SDValue TargetLowering::expandCLMUL(SDNode *Node,
                         !isOperationLegalOrCustom(ISD::SHL, VT) ||
                         !isOperationLegalOrCustom(ISD::XOR, VT) ||
                         !isOperationLegalOrCustom(ISD::AND, VT) ||
-                        !isOperationLegalOrCustom(ISD::SELECT, VT) ||
-                        !isOperationLegalOrCustomOrPromote(ISD::OR, VT))))
+                        !isOperationLegalOrCustom(ISD::SELECT, VT))))
     return SDValue();
 
   SDValue Res = DAG.getConstant(0, DL, VT);
   SDValue Zero = DAG.getConstant(0, DL, VT);
   SDValue One = DAG.getConstant(1, DL, VT);
-  for (unsigned i = 0; i < NumBitsPerElt-1; ++i) {
+  for (unsigned I = 0; I < NumBitsPerElt-1; ++I) {
     SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
-    SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, One, ISD::SETULT);
+    SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, Zero, ISD::SETNE);
     SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
     Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
-    V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
-    V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
-  }
-  // unroll last iteration to prevent dead nodes
-  SDValue LowBit = DAG.getNode(ISD::AND, DL, VT, V1, One);
-  SDValue LowBool = DAG.getSetCC(DL, SetCCType, LowBit, One, ISD::SETULT);
-  SDValue Pred = DAG.getNode(ISD::SELECT, DL, VT, LowBool, V2, Zero);
-  Res = DAG.getNode(ISD::XOR, DL, VT, Res, Pred);
+    if (I != NumBitsPerElt) {
+      V1 = DAG.getNode(ISD::SRL, DL, VT, V1, One);
+      V2 = DAG.getNode(ISD::SHL, DL, VT, V2, One);
+    }
+  }
   return Res;
 }