Skip to content

Commit e88374d

Browse files
LewisCrawfordLewisCrawford
committed
[DAGCombiner] Improve FMin/FMax DAGCombines
Add several improvements to DAGCombine patterns for fmin/fmax: * Fix incorrect results due to minimumnum not being marked as IsMin - e.g. nnan minimumnum(x, +inf) returned +inf instead of x * Fix incorrect results checking maximumnum for vecreduce patterns * Make maxnum/minnum return QNaN if one input is SNaN instead of X * Quiet SNaN inputs when propagating them e.g. maximum(x, SNaN) = QNaN * Update comments to mark when SNaN propagation is being ignored
1 parent 2f7252a commit e88374d

File tree

5 files changed

+264
-15
lines changed

5 files changed

+264
-15
lines changed

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp‎

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19323,8 +19323,10 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) {
1932319323
EVT VT = N->getValueType(0);
1932419324
const SDNodeFlags Flags = N->getFlags();
1932519325
unsigned Opc = N->getOpcode();
19326-
bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
19327-
bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM;
19326+
bool PropAllNaNsToQNaNs = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
19327+
bool PropOnlySNaNsToQNaNs = Opc == ISD::FMINNUM || Opc == ISD::FMAXNUM;
19328+
bool IsMin =
19329+
Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM || Opc == ISD::FMINIMUMNUM;
1932819330
SelectionDAG::FlagInserter FlagsInserter(DAG, N);
1932919331

1933019332
// Constant fold.
@@ -19339,34 +19341,53 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) {
1933919341
if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1)) {
1934019342
const APFloat &AF = N1CFP->getValueAPF();
1934119343

19342-
// minnum(X, nan) -> X
19343-
// maxnum(X, nan) -> X
19344-
// minimum(X, nan) -> nan
19345-
// maximum(X, nan) -> nan
19346-
if (AF.isNaN())
19347-
return PropagatesNaN ? N->getOperand(1) : N->getOperand(0);
19344+
// minnum(X, qnan) -> X
19345+
// maxnum(X, qnan) -> X
19346+
// minnum(X, snan) -> qnan
19347+
// maxnum(X, snan) -> qnan
19348+
// minimum(X, nan) -> qnan
19349+
// maximum(X, nan) -> qnan
19350+
// minimumnum(X, nan) -> X
19351+
// maximumnum(X, nan) -> X
19352+
if (AF.isNaN()) {
19353+
if (PropAllNaNsToQNaNs || (AF.isSignaling() && PropOnlySNaNsToQNaNs)) {
19354+
if (AF.isSignaling())
19355+
return DAG.getConstantFP(AF.makeQuiet(), SDLoc(N), VT);
19356+
return N->getOperand(1);
19357+
}
19358+
return N->getOperand(0);
19359+
}
1934819360

1934919361
// In the following folds, inf can be replaced with the largest finite
1935019362
// float, if the ninf flag is set.
1935119363
if (AF.isInfinity() || (Flags.hasNoInfs() && AF.isLargest())) {
19352-
// minnum(X, -inf) -> -inf
19353-
// maxnum(X, +inf) -> +inf
19364+
// minnum(X, -inf) -> -inf (ignoring sNaN -> qNaN propagation)
19365+
// maxnum(X, +inf) -> +inf (ignoring sNaN -> qNaN propagation)
1935419366
// minimum(X, -inf) -> -inf if nnan
1935519367
// maximum(X, +inf) -> +inf if nnan
19356-
if (IsMin == AF.isNegative() && (!PropagatesNaN || Flags.hasNoNaNs()))
19368+
// minimumnum(X, -inf) -> -inf
19369+
// maximumnum(X, +inf) -> +inf
19370+
if (IsMin == AF.isNegative() &&
19371+
(!PropAllNaNsToQNaNs || Flags.hasNoNaNs()))
1935719372
return N->getOperand(1);
1935819373

1935919374
// minnum(X, +inf) -> X if nnan
1936019375
// maxnum(X, -inf) -> X if nnan
19361-
// minimum(X, +inf) -> X
19362-
// maximum(X, -inf) -> X
19363-
if (IsMin != AF.isNegative() && (PropagatesNaN || Flags.hasNoNaNs()))
19376+
// minimum(X, +inf) -> X (ignoring quieting of sNaNs)
19377+
// maximum(X, -inf) -> X (ignoring quieting of sNaNs)
19378+
// minimumnum(X, +inf) -> X if nnan
19379+
// maximumnum(X, -inf) -> X if nnan
19380+
if (IsMin != AF.isNegative() && (PropAllNaNsToQNaNs || Flags.hasNoNaNs()))
1936419381
return N->getOperand(0);
1936519382
}
1936619383
}
1936719384

19385+
// There are no VECREDUCE variants of FMINIMUMNUM or FMAXIMUMNUM
19386+
if (Opc == ISD::FMINIMUMNUM || Opc == ISD::FMAXIMUMNUM)
19387+
return SDValue();
19388+
1936819389
if (SDValue SD = reassociateReduction(
19369-
PropagatesNaN
19390+
PropAllNaNsToQNaNs
1937019391
? (IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM)
1937119392
: (IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX),
1937219393
Opc, SDLoc(N), VT, N0, N1, Flags))

‎llvm/test/CodeGen/X86/fmaxnum.ll‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,5 +653,29 @@ define float @test_maxnum_const_nan(float %x) {
653653
ret float %r
654654
}
655655

656+
; nnan maxnum(X, -inf) -> X
657+
define float @test_maxnum_neg_inf_nnan(float %x) nounwind {
658+
; CHECK-LABEL: test_maxnum_neg_inf_nnan:
659+
; CHECK: # %bb.0:
660+
; CHECK-NEXT: retq
661+
%r = call nnan float @llvm.maxnum.f32(float %x, float 0xfff0000000000000)
662+
ret float %r
663+
}
664+
665+
; Test SNaN quieting
666+
define float @test_maxnum_snan(float %x) {
667+
; SSE-LABEL: test_maxnum_snan:
668+
; SSE: # %bb.0:
669+
; SSE-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
670+
; SSE-NEXT: retq
671+
;
672+
; AVX-LABEL: test_maxnum_snan:
673+
; AVX: # %bb.0:
674+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
675+
; AVX-NEXT: retq
676+
%r = call float @llvm.maxnum.f32(float 0x7ff4000000000000, float %x)
677+
ret float %r
678+
}
679+
656680
attributes #0 = { "no-nans-fp-math"="true" }
657681

‎llvm/test/CodeGen/X86/fminimum-fmaximum.ll‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2649,3 +2649,96 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
26492649
%r = call <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
26502650
ret <4 x bfloat> %r
26512651
}
2652+
2653+
; nnan minimum(X, +inf) -> X
2654+
define float @test_fminimum_inf_nnan(float %x) nounwind {
2655+
; SSE2-LABEL: test_fminimum_inf_nnan:
2656+
; SSE2: # %bb.0:
2657+
; SSE2-NEXT: retq
2658+
;
2659+
; AVX-LABEL: test_fminimum_inf_nnan:
2660+
; AVX: # %bb.0:
2661+
; AVX-NEXT: retq
2662+
;
2663+
; AVX10_2-LABEL: test_fminimum_inf_nnan:
2664+
; AVX10_2: # %bb.0:
2665+
; AVX10_2-NEXT: retq
2666+
;
2667+
; X86-LABEL: test_fminimum_inf_nnan:
2668+
; X86: # %bb.0:
2669+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2670+
; X86-NEXT: retl
2671+
%1 = call nnan float @llvm.minimum.f32(float %x, float 0x7ff0000000000000)
2672+
ret float %1
2673+
}
2674+
2675+
; nnan maximum(X, -inf) -> X
2676+
define float @test_fmaximum_neg_inf_nnan(float %x) nounwind {
2677+
; SSE2-LABEL: test_fmaximum_neg_inf_nnan:
2678+
; SSE2: # %bb.0:
2679+
; SSE2-NEXT: retq
2680+
;
2681+
; AVX-LABEL: test_fmaximum_neg_inf_nnan:
2682+
; AVX: # %bb.0:
2683+
; AVX-NEXT: retq
2684+
;
2685+
; AVX10_2-LABEL: test_fmaximum_neg_inf_nnan:
2686+
; AVX10_2: # %bb.0:
2687+
; AVX10_2-NEXT: retq
2688+
;
2689+
; X86-LABEL: test_fmaximum_neg_inf_nnan:
2690+
; X86: # %bb.0:
2691+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2692+
; X86-NEXT: retl
2693+
%1 = call nnan float @llvm.maximum.f32(float %x, float 0xfff0000000000000)
2694+
ret float %1
2695+
}
2696+
2697+
; Test SNaN quieting
2698+
define float @test_fmaximum_snan(float %x) {
2699+
; SSE2-LABEL: test_fmaximum_snan:
2700+
; SSE2: # %bb.0:
2701+
; SSE2-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2702+
; SSE2-NEXT: retq
2703+
;
2704+
; AVX-LABEL: test_fmaximum_snan:
2705+
; AVX: # %bb.0:
2706+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2707+
; AVX-NEXT: retq
2708+
;
2709+
; AVX10_2-LABEL: test_fmaximum_snan:
2710+
; AVX10_2: # %bb.0:
2711+
; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2712+
; AVX10_2-NEXT: retq
2713+
;
2714+
; X86-LABEL: test_fmaximum_snan:
2715+
; X86: # %bb.0:
2716+
; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
2717+
; X86-NEXT: retl
2718+
%1 = tail call float @llvm.maximum.f32(float 0x7ff4000000000000, float %x)
2719+
ret float %1
2720+
}
2721+
2722+
define float @test_fminimum_snan(float %x) {
2723+
; SSE2-LABEL: test_fminimum_snan:
2724+
; SSE2: # %bb.0:
2725+
; SSE2-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2726+
; SSE2-NEXT: retq
2727+
;
2728+
; AVX-LABEL: test_fminimum_snan:
2729+
; AVX: # %bb.0:
2730+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2731+
; AVX-NEXT: retq
2732+
;
2733+
; AVX10_2-LABEL: test_fminimum_snan:
2734+
; AVX10_2: # %bb.0:
2735+
; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2736+
; AVX10_2-NEXT: retq
2737+
;
2738+
; X86-LABEL: test_fminimum_snan:
2739+
; X86: # %bb.0:
2740+
; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
2741+
; X86-NEXT: retl
2742+
%1 = tail call float @llvm.minimum.f32(float 0x7ff4000000000000, float %x)
2743+
ret float %1
2744+
}

‎llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2479,3 +2479,90 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
24792479
%r = call <4 x bfloat> @llvm.maximumnum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
24802480
ret <4 x bfloat> %r
24812481
}
2482+
2483+
; nnan minimumnum(X, +inf) -> X
2484+
define float @test_fminimumnum_inf_nnan(float %x) nounwind {
2485+
; SSE2-LABEL: test_fminimumnum_inf_nnan:
2486+
; SSE2: # %bb.0:
2487+
; SSE2-NEXT: retq
2488+
;
2489+
; AVX-LABEL: test_fminimumnum_inf_nnan:
2490+
; AVX: # %bb.0:
2491+
; AVX-NEXT: retq
2492+
;
2493+
; AVX10_2-LABEL: test_fminimumnum_inf_nnan:
2494+
; AVX10_2: # %bb.0:
2495+
; AVX10_2-NEXT: retq
2496+
;
2497+
; X86-LABEL: test_fminimumnum_inf_nnan:
2498+
; X86: # %bb.0:
2499+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2500+
; X86-NEXT: retl
2501+
%1 = call nnan float @llvm.minimumnum.f32(float %x, float 0x7ff0000000000000)
2502+
ret float %1
2503+
}
2504+
2505+
; nnan maximumnum(X, -inf) -> X
2506+
define float @test_fmaximumnum_neg_inf_nnan(float %x) nounwind {
2507+
; SSE2-LABEL: test_fmaximumnum_neg_inf_nnan:
2508+
; SSE2: # %bb.0:
2509+
; SSE2-NEXT: retq
2510+
;
2511+
; AVX-LABEL: test_fmaximumnum_neg_inf_nnan:
2512+
; AVX: # %bb.0:
2513+
; AVX-NEXT: retq
2514+
;
2515+
; AVX10_2-LABEL: test_fmaximumnum_neg_inf_nnan:
2516+
; AVX10_2: # %bb.0:
2517+
; AVX10_2-NEXT: retq
2518+
;
2519+
; X86-LABEL: test_fmaximumnum_neg_inf_nnan:
2520+
; X86: # %bb.0:
2521+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2522+
; X86-NEXT: retl
2523+
%1 = call nnan float @llvm.maximumnum.f32(float %x, float 0xfff0000000000000)
2524+
ret float %1
2525+
}
2526+
2527+
; Test we propagate the non-NaN arg, even if one arg is SNaN
2528+
define float @test_fmaximumnum_snan(float %x) {
2529+
; SSE2-LABEL: test_fmaximumnum_snan:
2530+
; SSE2: # %bb.0:
2531+
; SSE2-NEXT: retq
2532+
;
2533+
; AVX-LABEL: test_fmaximumnum_snan:
2534+
; AVX: # %bb.0:
2535+
; AVX-NEXT: retq
2536+
;
2537+
; AVX10_2-LABEL: test_fmaximumnum_snan:
2538+
; AVX10_2: # %bb.0:
2539+
; AVX10_2-NEXT: retq
2540+
;
2541+
; X86-LABEL: test_fmaximumnum_snan:
2542+
; X86: # %bb.0:
2543+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2544+
; X86-NEXT: retl
2545+
%1 = tail call float @llvm.maximumnum.f32(float 0x7ff4000000000000, float %x)
2546+
ret float %1
2547+
}
2548+
2549+
define float @test_fminimumnum_snan(float %x) {
2550+
; SSE2-LABEL: test_fminimumnum_snan:
2551+
; SSE2: # %bb.0:
2552+
; SSE2-NEXT: retq
2553+
;
2554+
; AVX-LABEL: test_fminimumnum_snan:
2555+
; AVX: # %bb.0:
2556+
; AVX-NEXT: retq
2557+
;
2558+
; AVX10_2-LABEL: test_fminimumnum_snan:
2559+
; AVX10_2: # %bb.0:
2560+
; AVX10_2-NEXT: retq
2561+
;
2562+
; X86-LABEL: test_fminimumnum_snan:
2563+
; X86: # %bb.0:
2564+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2565+
; X86-NEXT: retl
2566+
%1 = tail call float @llvm.minimumnum.f32(float 0x7ff4000000000000, float %x)
2567+
ret float %1
2568+
}

‎llvm/test/CodeGen/X86/fminnum.ll‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,5 +653,29 @@ define float @test_minnum_const_nan(float %x) {
653653
ret float %r
654654
}
655655

656+
; nnan minnum(X, +inf) -> X
657+
define float @test_minnum_inf_nnan(float %x) nounwind {
658+
; CHECK-LABEL: test_minnum_inf_nnan:
659+
; CHECK: # %bb.0:
660+
; CHECK-NEXT: retq
661+
%r = call nnan float @llvm.minnum.f32(float %x, float 0x7ff0000000000000)
662+
ret float %r
663+
}
664+
665+
; Test SNaN quieting
666+
define float @test_minnum_snan(float %x) {
667+
; SSE-LABEL: test_minnum_snan:
668+
; SSE: # %bb.0:
669+
; SSE-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
670+
; SSE-NEXT: retq
671+
;
672+
; AVX-LABEL: test_minnum_snan:
673+
; AVX: # %bb.0:
674+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
675+
; AVX-NEXT: retq
676+
%r = call float @llvm.minnum.f32(float 0x7ff4000000000000, float %x)
677+
ret float %r
678+
}
679+
656680
attributes #0 = { "no-nans-fp-math"="true" }
657681

0 commit comments

Comments
 (0)