@@ -19380,43 +19380,59 @@ class HorizontalReduction {
1938019380 /// Creates reduction operation with the current opcode.
1938119381 static Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
1938219382 Value *RHS, const Twine &Name, bool UseSelect) {
19383- if (UseSelect) {
19384- if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind)) {
19385- CmpInst::Predicate Pred = llvm::getMinMaxReductionPredicate(Kind);
19386- Value *Cmp = Builder.CreateCmp(Pred, LHS, RHS, Name);
19387- return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19388- }
19389- if ((Kind == RecurKind::Or || Kind == RecurKind::And) &&
19390- LHS->getType() == CmpInst::makeCmpResultType(LHS->getType())) {
19391- Value *TrueVal = Kind == RecurKind::Or ? Builder.getTrue() : RHS;
19392- Value *FalseVal = Kind == RecurKind::Or ? RHS : Builder.getFalse();
19393- return Builder.CreateSelect(LHS, TrueVal, FalseVal, Name);
19394- }
19395- }
19396-
19383+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
1939719384 switch (Kind) {
1939819385 case RecurKind::Or:
19386+ if (UseSelect &&
19387+ LHS->getType() == CmpInst::makeCmpResultType(LHS->getType()))
19388+ return Builder.CreateSelect(LHS, Builder.getTrue(), RHS, Name);
19389+ return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
19390+ Name);
1939919391 case RecurKind::And:
19392+ if (UseSelect &&
19393+ LHS->getType() == CmpInst::makeCmpResultType(LHS->getType()))
19394+ return Builder.CreateSelect(LHS, RHS, Builder.getFalse(), Name);
19395+ return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
19396+ Name);
1940019397 case RecurKind::Add:
1940119398 case RecurKind::Mul:
1940219399 case RecurKind::Xor:
1940319400 case RecurKind::FAdd:
19404- case RecurKind::FMul: {
19405- unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
19401+ case RecurKind::FMul:
1940619402 return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
1940719403 Name);
19408- }
1940919404 case RecurKind::FMax:
19405+ return Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS);
1941019406 case RecurKind::FMin:
19407+ return Builder.CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS);
1941119408 case RecurKind::FMaximum:
19409+ return Builder.CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS);
1941219410 case RecurKind::FMinimum:
19411+ return Builder.CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS);
1941319412 case RecurKind::SMax:
19413+ if (UseSelect) {
19414+ Value *Cmp = Builder.CreateICmpSGT(LHS, RHS, Name);
19415+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19416+ }
19417+ return Builder.CreateBinaryIntrinsic(Intrinsic::smax, LHS, RHS);
1941419418 case RecurKind::SMin:
19419+ if (UseSelect) {
19420+ Value *Cmp = Builder.CreateICmpSLT(LHS, RHS, Name);
19421+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19422+ }
19423+ return Builder.CreateBinaryIntrinsic(Intrinsic::smin, LHS, RHS);
1941519424 case RecurKind::UMax:
19416- case RecurKind::UMin: {
19417- Intrinsic::ID Id = llvm::getMinMaxReductionIntrinsicOp(Kind);
19418- return Builder.CreateBinaryIntrinsic(Id, LHS, RHS);
19419- }
19425+ if (UseSelect) {
19426+ Value *Cmp = Builder.CreateICmpUGT(LHS, RHS, Name);
19427+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19428+ }
19429+ return Builder.CreateBinaryIntrinsic(Intrinsic::umax, LHS, RHS);
19430+ case RecurKind::UMin:
19431+ if (UseSelect) {
19432+ Value *Cmp = Builder.CreateICmpULT(LHS, RHS, Name);
19433+ return Builder.CreateSelect(Cmp, LHS, RHS, Name);
19434+ }
19435+ return Builder.CreateBinaryIntrinsic(Intrinsic::umin, LHS, RHS);
1942019436 default:
1942119437 llvm_unreachable("Unknown reduction operation.");
1942219438 }
0 commit comments