[X86] Check that the type is integer before calling isUnsignedIntSetCC in combineExtSetcc. (#128263)

SETULT can be an unsigned less than integer compare or a unordered less
than FP compare. We need to check the VT to distinguish them.

Fixes on of the issues from #128237.

Author: topperc

llvm/lib/Target/X86/X86ISelLowering.cpp

@@ -55368,14 +55368,15 @@ static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
   if (Size > 256 && Subtarget.useAVX512Regs())
     return SDValue();
 
+  EVT N00VT = N0.getOperand(0).getValueType();
+
   // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
   // that's the only integer compares with we have.
   ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
-  if (ISD::isUnsignedIntSetCC(CC))
+  if (N00VT.isInteger() && ISD::isUnsignedIntSetCC(CC))
     return SDValue();
 
   // Only do this combine if the extension will be fully consumed by the setcc.
-  EVT N00VT = N0.getOperand(0).getValueType();
   EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
   if (Size != MatchingVecType.getSizeInBits())
     return SDValue();

llvm/test/CodeGen/X86/v8i1-masks.ll

@@ -150,21 +150,17 @@ define void @neg_masks(ptr %a, ptr %b, ptr %c) nounwind uwtable noinline ssp {
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX512-NEXT:    vmovups (%ecx), %ymm0
-; X86-AVX512-NEXT:    vcmpnltps (%eax), %ymm0, %k1
-; X86-AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
-; X86-AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
-; X86-AVX512-NEXT:    vpsrld $31, %ymm0, %ymm0
+; X86-AVX512-NEXT:    vcmpnltps (%eax), %ymm0, %ymm0
+; X86-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
 ; X86-AVX512-NEXT:    vmovdqa %ymm0, (%eax)
 ; X86-AVX512-NEXT:    vzeroupper
 ; X86-AVX512-NEXT:    retl
 ;
 ; X64-AVX512-LABEL: neg_masks:
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vmovups (%rsi), %ymm0
-; X64-AVX512-NEXT:    vcmpnltps (%rdi), %ymm0, %k1
-; X64-AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
-; X64-AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
-; X64-AVX512-NEXT:    vpsrld $31, %ymm0, %ymm0
+; X64-AVX512-NEXT:    vcmpnltps (%rdi), %ymm0, %ymm0
+; X64-AVX512-NEXT:    vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vmovdqa %ymm0, (%rax)
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq