[SelectionDAG] Fix AArch64 machine verifier bug when expanding LOOP_DEPENDENCE_MASK

We did not ensure new opcodes like mi/pl were filtered out when swapping, and TargetConstant nodes don't match TableGen ImmLeaf patterns during instruction selection. When this zero constant flows into the AArch64 CCMP formation code, the machine verifier hit an assertion in expensive checks.

Author: AZero13

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp

@@ -1829,7 +1829,7 @@ SDValue VectorLegalizer::ExpandLOOP_DEPENDENCE_MASK(SDNode *N) {
   // If the difference is positive then some elements may alias
   EVT CmpVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
                                      Diff.getValueType());
-  SDValue Zero = DAG.getTargetConstant(0, DL, PtrVT);
+  SDValue Zero = DAG.getConstant(0, DL, PtrVT);
   SDValue Cmp = DAG.getSetCC(DL, CmpVT, Diff, Zero,
                              IsReadAfterWrite ? ISD::SETEQ : ISD::SETLE);
 

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

@@ -413,7 +413,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_LOOP_DEPENDENCE_MASK(SDNode *N) {
   SDValue Diff = DAG.getNode(ISD::SUB, DL, PtrVT, SinkValue, SourceValue);
   EVT CmpVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
                                      Diff.getValueType());
-  SDValue Zero = DAG.getTargetConstant(0, DL, PtrVT);
+  SDValue Zero = DAG.getConstant(0, DL, PtrVT);
   return DAG.getNode(ISD::OR, DL, CmpVT,
                      DAG.getSetCC(DL, CmpVT, Diff, EltSize, ISD::SETGE),
                      DAG.getSetCC(DL, CmpVT, Diff, Zero, ISD::SETEQ));

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

@@ -26013,9 +26013,12 @@ static SDValue reassociateCSELOperandsForCSE(SDNode *N, SelectionDAG &DAG) {
     // Try again with the operands of the SUBS instruction and the condition
     // swapped. Due to canonicalization, this only helps for non-constant
     // operands of the SUBS instruction.
-    std::swap(CmpOpToMatch, CmpOpOther);
-    if (SDValue R = Fold(getSwappedCondition(CC), CmpOpToMatch, CmpOpToMatch))
-      return R;
+    auto NewCC = getSwappedCondition(CC);
+    if (NewCC != AArch64CC::AL) {
+      std::swap(CmpOpToMatch, CmpOpOther);
+      if (SDValue R = Fold(NewCC, CmpOpToMatch, CmpOpToMatch))
+        return R;
+    }
     return SDValue();
   }
 

llvm/test/CodeGen/AArch64/alias_mask.ll

@@ -793,9 +793,8 @@ define <1 x i1> @whilewr_8_scalarize(ptr %a, ptr %b) {
 ; CHECK-LABEL: whilewr_8_scalarize:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub x8, x1, x0
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    ccmp x8, #0, #4, le
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    cmn x8, #1
+; CHECK-NEXT:    cset w0, gt
 ; CHECK-NEXT:    ret
 entry:
   %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 1)
@@ -845,9 +844,8 @@ define <1 x i1> @whilerw_8_scalarize(ptr %a, ptr %b) {
 ; CHECK-LABEL: whilerw_8_scalarize:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    sub x8, x1, x0
-; CHECK-NEXT:    cmp x8, #0
-; CHECK-NEXT:    ccmp x8, #0, #4, le
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    cmn x8, #1
+; CHECK-NEXT:    cset w0, gt
 ; CHECK-NEXT:    ret
 entry:
   %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 1)

llvm/test/CodeGen/AArch64/loop-dependence-mask-ccmp.ll

@@ -19,32 +19,34 @@
 ; Assembly output would look identical for both, so we must check machine IR.
 
 define <1 x i1> @test_war_mask_ccmp(ptr %a, ptr %b) {
-; CHECK-LABEL: name: test_war_mask_ccmp
-; CHECK: bb.0.entry:
-; CHECK:   [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
-; CHECK:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
-; CHECK:   [[SUBSXrr:%[0-9]+]]:gpr64common = SUBSXrr [[COPY1]], [[COPY]], implicit-def dead $nzcv
-; CHECK:   [[SUBSXri:%[0-9]+]]:gpr64 = SUBSXri [[SUBSXrr]], 0, 0, implicit-def $nzcv
-; CHECK:   CCMPXi [[SUBSXrr]], 0, 4, 13, implicit-def $nzcv, implicit $nzcv
-; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 1, implicit $nzcv
-; CHECK:   $w0 = COPY [[CSINCWr]]
-; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-LABEL: name: test_war_mask_ccmp
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $x0, $x1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-NEXT:   [[SUBSXrr:%[0-9]+]]:gpr64common = SUBSXrr [[COPY]], [[COPY1]], implicit-def dead $nzcv
+  ; CHECK-NEXT:   [[ADDSXri:%[0-9]+]]:gpr64 = ADDSXri killed [[SUBSXrr]], 1, 0, implicit-def $nzcv
+  ; CHECK-NEXT:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 13, implicit $nzcv
+  ; CHECK-NEXT:   $w0 = COPY [[CSINCWr]]
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
 entry:
   %0 = call <1 x i1> @llvm.loop.dependence.war.mask.v1i1(ptr %a, ptr %b, i64 1)
   ret <1 x i1> %0
 }
 
 define <1 x i1> @test_raw_mask_ccmp(ptr %a, ptr %b) {
-; CHECK-LABEL: name: test_raw_mask_ccmp
-; CHECK: bb.0.entry:
-; CHECK:   [[COPY1:%[0-9]+]]:gpr64 = COPY $x1
-; CHECK:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
-; CHECK:   [[SUBSXrr:%[0-9]+]]:gpr64common = SUBSXrr [[COPY1]], [[COPY]], implicit-def dead $nzcv
-; CHECK:   [[SUBSXri:%[0-9]+]]:gpr64 = SUBSXri [[SUBSXrr]], 0, 0, implicit-def $nzcv
-; CHECK:   CCMPXi [[SUBSXrr]], 0, 4, 13, implicit-def $nzcv, implicit $nzcv
-; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 1, implicit $nzcv
-; CHECK:   $w0 = COPY [[CSINCWr]]
-; CHECK:   RET_ReallyLR implicit $w0
+  ; CHECK-LABEL: name: test_raw_mask_ccmp
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $x0, $x1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr64 = COPY $x1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK-NEXT:   [[SUBSXrr:%[0-9]+]]:gpr64common = SUBSXrr [[COPY]], [[COPY1]], implicit-def dead $nzcv
+  ; CHECK-NEXT:   [[ADDSXri:%[0-9]+]]:gpr64 = ADDSXri killed [[SUBSXrr]], 1, 0, implicit-def $nzcv
+  ; CHECK-NEXT:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 13, implicit $nzcv
+  ; CHECK-NEXT:   $w0 = COPY [[CSINCWr]]
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
 entry:
   %0 = call <1 x i1> @llvm.loop.dependence.raw.mask.v1i1(ptr %a, ptr %b, i64 1)
   ret <1 x i1> %0