[x86] Lower arithmetic vector reductions

sutajo · sutajo · commit 376965edaa4a · 2025-06-19T01:06:34.000+02:00
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1058,6 +1058,11 @@ namespace llvm {
     /// functions.
     bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
                                             const MachineFunction &MF);
+
+    /// True if the target supports a fast implementation for the specific
+    /// operation and vector type combination.
+    bool isVectorReductionFast(const X86Subtarget &Subtarget, ISD::NodeType VectorOp, MVT VT);
+
   } // end namespace X86
 
   //===--------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6576,21 +6576,41 @@ X86TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
 }
 
 bool llvm::X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
+  ISD::NodeType OpCode;
   switch (II->getIntrinsicID()) {
-  default:
-    return true;
-
+  case Intrinsic::vector_reduce_add:
+    OpCode = ISD::VECREDUCE_ADD;
+    break;
+  case Intrinsic::vector_reduce_fadd:
+    OpCode = ISD::VECREDUCE_FADD;
+    break;
+  case Intrinsic::vector_reduce_mul:
+    OpCode = ISD::VECREDUCE_MUL;
+    break;
   case Intrinsic::vector_reduce_umin:
+    OpCode = ISD::VECREDUCE_UMIN;
+    break;
   case Intrinsic::vector_reduce_umax:
+    OpCode = ISD::VECREDUCE_UMAX;
+    break;
   case Intrinsic::vector_reduce_smin:
+    OpCode = ISD::VECREDUCE_SMIN;
+    break;
   case Intrinsic::vector_reduce_smax:
-    auto *VType = cast<FixedVectorType>(II->getOperand(0)->getType());
-    auto SType = VType->getScalarType();
-    bool CanUsePHMINPOSUW = ST->hasSSE41() && II->getType() == SType &&
-                            (VType->getPrimitiveSizeInBits() % 128) == 0 &&
-                            (SType->isIntegerTy(8) || SType->isIntegerTy(16));
-    return !CanUsePHMINPOSUW;
+    OpCode = ISD::VECREDUCE_SMAX;
+    break;
+
+  default:
+    return true;
   }
+
+  auto *VType = dyn_cast<FixedVectorType>(
+      II->getOperand(II->getIntrinsicID() == Intrinsic::vector_reduce_fadd ? 1
+                                                                           : 0)
+          ->getType());
+  auto VT = EVT::getEVT(VType).getSimpleVT();
+
+  return !X86::isVectorReductionFast(*ST, OpCode, VT);
 }
 
 bool X86TTIImpl::prefersVectorizedAddressing() const {
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -833,34 +833,34 @@ define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4
 ;
 ; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32:
 ; SSSE3-FAST:       # %bb.0:
-; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
-; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm4
-; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm5
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm0[1]
-; SSSE3-FAST-NEXT:    addss %xmm4, %xmm5
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSSE3-FAST-NEXT:    addss %xmm5, %xmm0
-; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm5
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
-; SSSE3-FAST-NEXT:    addss %xmm4, %xmm5
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; SSSE3-FAST-NEXT:    addss %xmm5, %xmm1
-; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm4
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
-; SSSE3-FAST-NEXT:    addss %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; SSSE3-FAST-NEXT:    addss %xmm4, %xmm2
-; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm1
-; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm1
 ; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm4
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
-; SSSE3-FAST-NEXT:    addss %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm4 = xmm4[3,3],xmm3[3,3]
+; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm5
+; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm3[1]
+; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm6
+; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm6 = xmm6[3,3],xmm2[3,3]
+; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm7
+; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm7 = xmm7[1],xmm2[1]
+; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm8
+; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm8 = xmm8[3,3],xmm1[3,3]
+; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm9
+; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm9 = xmm9[1],xmm1[1]
+; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm10
+; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm10 = xmm10[3,3],xmm0[3,3]
+; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm11
+; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm11 = xmm11[1],xmm0[1]
+; SSSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT:    addss %xmm11, %xmm0
+; SSSE3-FAST-NEXT:    addss %xmm10, %xmm0
+; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm1
+; SSSE3-FAST-NEXT:    addss %xmm9, %xmm1
+; SSSE3-FAST-NEXT:    addss %xmm8, %xmm1
+; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm2
+; SSSE3-FAST-NEXT:    addss %xmm7, %xmm2
+; SSSE3-FAST-NEXT:    addss %xmm6, %xmm2
+; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm3
+; SSSE3-FAST-NEXT:    addss %xmm5, %xmm3
 ; SSSE3-FAST-NEXT:    addss %xmm4, %xmm3
 ; SSSE3-FAST-NEXT:    unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
 ; SSSE3-FAST-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
@@ -899,28 +899,28 @@ define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4
 ;
 ; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32:
 ; AVX-FAST:       # %bb.0:
-; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm4
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm5 = xmm0[1,0]
-; AVX-FAST-NEXT:    vaddss %xmm5, %xmm4, %xmm4
-; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX-FAST-NEXT:    vaddss %xmm0, %xmm4, %xmm0
-; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm4
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm5 = xmm1[1,0]
-; AVX-FAST-NEXT:    vaddss %xmm5, %xmm4, %xmm4
-; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX-FAST-NEXT:    vaddss %xmm1, %xmm4, %xmm1
+; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[3,3,3,3]
+; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm5 = xmm3[1,0]
+; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm6 = xmm2[3,3,3,3]
+; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm7 = xmm2[1,0]
+; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm8 = xmm1[3,3,3,3]
+; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm9 = xmm1[1,0]
+; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm10 = xmm0[3,3,3,3]
+; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm11 = xmm0[1,0]
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vaddss %xmm0, %xmm11, %xmm0
+; AVX-FAST-NEXT:    vaddss %xmm0, %xmm10, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vaddss %xmm1, %xmm9, %xmm1
+; AVX-FAST-NEXT:    vaddss %xmm1, %xmm8, %xmm1
 ; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
 ; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm2, %xmm1
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm4 = xmm2[1,0]
-; AVX-FAST-NEXT:    vaddss %xmm4, %xmm1, %xmm1
-; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vaddss %xmm7, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vaddss %xmm6, %xmm1, %xmm1
 ; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
 ; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm3, %xmm1
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm2 = xmm3[1,0]
-; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[3,3,3,3]
-; AVX-FAST-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vaddss %xmm5, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vaddss %xmm4, %xmm1, %xmm1
 ; AVX-FAST-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
 ; AVX-FAST-NEXT:    retq
   %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
@@ -964,21 +964,9 @@ define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float
 ;
 ; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
 ; SSSE3-FAST:       # %bb.0:
-; SSSE3-FAST-NEXT:    movaps %xmm0, %xmm4
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
-; SSSE3-FAST-NEXT:    addps %xmm4, %xmm0
-; SSSE3-FAST-NEXT:    movaps %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
-; SSSE3-FAST-NEXT:    addps %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    haddps %xmm4, %xmm0
-; SSSE3-FAST-NEXT:    movaps %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
-; SSSE3-FAST-NEXT:    addps %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    movaps %xmm3, %xmm2
-; SSSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
-; SSSE3-FAST-NEXT:    addps %xmm3, %xmm2
-; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; SSSE3-FAST-NEXT:    haddps %xmm3, %xmm2
+; SSSE3-FAST-NEXT:    haddps %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    haddps %xmm2, %xmm0
 ; SSSE3-FAST-NEXT:    retq
 ;
 ; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:
@@ -1002,17 +990,9 @@ define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float
 ;
 ; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
 ; AVX-FAST:       # %bb.0:
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm4 = xmm0[1,0]
-; AVX-FAST-NEXT:    vaddps %xmm4, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm4 = xmm1[1,0]
-; AVX-FAST-NEXT:    vaddps %xmm4, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vhaddps %xmm3, %xmm2, %xmm2
 ; AVX-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm1 = xmm2[1,0]
-; AVX-FAST-NEXT:    vaddps %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT:    vshufpd {{.*#+}} xmm2 = xmm3[1,0]
-; AVX-FAST-NEXT:    vaddps %xmm2, %xmm3, %xmm2
-; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX-FAST-NEXT:    vhaddps %xmm2, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    retq
   %5 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
   %6 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)
@@ -1051,17 +1031,9 @@ define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32
 ;
 ; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:
 ; SSSE3-FAST:       # %bb.0:
-; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; SSSE3-FAST-NEXT:    paddd %xmm4, %xmm0
-; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; SSSE3-FAST-NEXT:    paddd %xmm1, %xmm4
-; SSSE3-FAST-NEXT:    phaddd %xmm4, %xmm0
-; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; SSSE3-FAST-NEXT:    paddd %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; SSSE3-FAST-NEXT:    paddd %xmm3, %xmm2
-; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm1
-; SSSE3-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; SSSE3-FAST-NEXT:    phaddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT:    phaddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT:    phaddd %xmm2, %xmm0
 ; SSSE3-FAST-NEXT:    retq
 ;
 ; AVX1-SLOW-LABEL: reduction_sum_v4i32_v4i32:
@@ -1089,17 +1061,9 @@ define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32
 ;
 ; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:
 ; AVX-FAST:       # %bb.0:
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX-FAST-NEXT:    vpaddd %xmm4, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; AVX-FAST-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; AVX-FAST-NEXT:    vphaddd %xmm3, %xmm2, %xmm2
 ; AVX-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; AVX-FAST-NEXT:    vpaddd %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; AVX-FAST-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
-; AVX-FAST-NEXT:    vphaddd %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX-FAST-NEXT:    vphaddd %xmm2, %xmm0, %xmm0
 ; AVX-FAST-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: reduction_sum_v4i32_v4i32:
diff --git a/llvm/test/CodeGen/X86/optimize-reduction.ll b/llvm/test/CodeGen/X86/optimize-reduction.ll
@@ -89,6 +89,7 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
 ; AVX2-NEXT:    vphaddw %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vphaddw %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vmovd %xmm2, %eax
+; AVX2-NEXT:    vmovd %eax, %xmm2
 ; AVX2-NEXT:    vpbroadcastw %xmm2, %ymm2
 ; AVX2-NEXT:    vpcmpeqw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
@@ -1006,20 +1006,34 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) {
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v16i16_v16i8:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT:    vzeroupper
-; AVX1-NEXT:    retq
+; AVX1-SLOW-LABEL: test_v16i16_v16i8:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: test_v16i16_v16i8:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-LABEL: test_v16i16_v16i8:
 ; AVX2:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
@@ -220,13 +220,21 @@ define i32 @test_v4i32(<4 x i8> %a0) {
 ; SSE41-NEXT:    movd %xmm0, %eax
 ; SSE41-NEXT:    retq
 ;
-; AVX1-LABEL: test_v4i32:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
-; AVX1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    retq
+; AVX1-SLOW-LABEL: test_v4i32:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; AVX1-SLOW-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: test_v4i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    retq
 ;
 ; AVX2-LABEL: test_v4i32:
 ; AVX2:       # %bb.0:
@@ -257,12 +265,37 @@ define i32 @test_v8i32_v8i8(<8 x i8> %a0) {
 ; SSE-NEXT:    movd %xmm1, %eax
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test_v8i32_v8i8:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vmovd %xmm0, %eax
-; AVX-NEXT:    retq
+; AVX1-SLOW-LABEL: test_v8i32_v8i8:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: test_v8i32_v8i8:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: test_v8i32_v8i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test_v8i32_v8i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
   %1 = zext <8 x i8> %a0 to <8 x i32>
   %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
   ret i32 %2
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fadd.ll b/llvm/test/CodeGen/X86/vector-reduce-fadd.ll
