[SLP]Fix a check for the whole register use

Need to check the value type, not the return type, of the instructions,
when doing the analysis for the whole register use to prevent a compiler
crash.

Fixes #133751

Author: alexey-bataev

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -8854,7 +8854,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
       // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
       if ((UserTreeIdx.UserTE &&
            UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(*TTI)) ||
-          !hasFullVectorsOrPowerOf2(*TTI, VL.front()->getType(), VL.size())) {
+          !hasFullVectorsOrPowerOf2(*TTI, getValueType(VL.front()),
+                                    VL.size())) {
         LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
                              "for nodes with padding.\n");
         auto Invalid = ScheduleBundle::invalid();

llvm/test/Transforms/SLPVectorizer/X86/cmp-values-non-full-registers.ll

@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-grtev4-linux-gnu < %s | FileCheck %s
+
+define void @test() {
+; CHECK-LABEL: define void @test() {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[BB1:.*]], label %[[BB2:.*]]
+; CHECK:       [[BB2]]:
+; CHECK-NEXT:    [[CMP14_US:%.*]] = fcmp ogt x86_fp80 0xK00000000000000000000, 0xK00000000000000000000
+; CHECK-NEXT:    [[IMAX_2_US:%.*]] = select i1 [[CMP14_US]], i64 0, i64 0
+; CHECK-NEXT:    [[JMAX_2_US:%.*]] = select i1 [[CMP14_US]], i64 0, i64 0
+; CHECK-NEXT:    [[CMP13_US:%.*]] = fcmp olt x86_fp80 0xK00000000000000000000, 0xK00000000000000000000
+; CHECK-NEXT:    [[IMIN_2_US:%.*]] = select i1 [[CMP13_US]], i64 0, i64 0
+; CHECK-NEXT:    [[JMIN_2_US:%.*]] = select i1 [[CMP13_US]], i64 0, i64 0
+; CHECK-NEXT:    br label %[[BB1]]
+; CHECK:       [[BB1]]:
+; CHECK-NEXT:    [[STOREMERGE64:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IMIN_2_US]], %[[BB2]] ]
+; CHECK-NEXT:    [[STOREMERGE63:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[JMIN_2_US]], %[[BB2]] ]
+; CHECK-NEXT:    [[STOREMERGE62:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IMAX_2_US]], %[[BB2]] ]
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[JMAX_2_US]], %[[BB2]] ]
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 false, label %bb1, label %bb2
+
+bb2:
+  %cmp14.us = fcmp ogt x86_fp80 0xK00000000000000000000, 0xK00000000000000000000
+  %imax.2.us = select i1 %cmp14.us, i64 0, i64 0
+  %jmax.2.us = select i1 %cmp14.us, i64 0, i64 0
+  %cmp13.us = fcmp olt x86_fp80 0xK00000000000000000000, 0xK00000000000000000000
+  %imin.2.us = select i1 %cmp13.us, i64 0, i64 0
+  %jmin.2.us = select i1 %cmp13.us, i64 0, i64 0
+  br label %bb1
+
+bb1:
+  %storemerge64 = phi i64 [ 0, %entry ], [ %imin.2.us, %bb2 ]
+  %storemerge63 = phi i64 [ 0, %entry ], [ %jmin.2.us, %bb2 ]
+  %storemerge62 = phi i64 [ 0, %entry ], [ %imax.2.us, %bb2 ]
+  %storemerge = phi i64 [ 0, %entry ], [ %jmax.2.us, %bb2 ]
+  ret void
+}