Skip to content

Commit 7ba3107

Browse files
LewisCrawfordsvkeerthy
LewisCrawford
authored and
svkeerthy
committed
[DAGCombiner] Improve FMin/FMax DAGCombines (#161352)
Add several improvements to DAGCombine patterns for fmin/fmax: * Fix incorrect results due to minimumnum not being marked as IsMin - e.g. nnan minimumnum(x, +inf) returned +inf instead of x * Fix incorrect results checking maximumnum for vecreduce patterns * Make maxnum/minnum return QNaN if one input is SNaN instead of X * Quiet SNaN inputs when propagating them e.g. maximum(x, SNaN) = QNaN * Update comments to mark when SNaN propagation is being ignored
1 parent eb6207b commit 7ba3107

File tree

5 files changed

+316
-25
lines changed

5 files changed

+316
-25
lines changed

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp‎

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19340,8 +19340,10 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) {
1934019340
EVT VT = N->getValueType(0);
1934119341
const SDNodeFlags Flags = N->getFlags();
1934219342
unsigned Opc = N->getOpcode();
19343-
bool PropagatesNaN = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
19344-
bool IsMin = Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM;
19343+
bool PropAllNaNsToQNaNs = Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM;
19344+
bool PropOnlySNaNsToQNaNs = Opc == ISD::FMINNUM || Opc == ISD::FMAXNUM;
19345+
bool IsMin =
19346+
Opc == ISD::FMINNUM || Opc == ISD::FMINIMUM || Opc == ISD::FMINIMUMNUM;
1934519347
SelectionDAG::FlagInserter FlagsInserter(DAG, N);
1934619348

1934719349
// Constant fold.
@@ -19356,34 +19358,53 @@ SDValue DAGCombiner::visitFMinMax(SDNode *N) {
1935619358
if (const ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1)) {
1935719359
const APFloat &AF = N1CFP->getValueAPF();
1935819360

19359-
// minnum(X, nan) -> X
19360-
// maxnum(X, nan) -> X
19361-
// minimum(X, nan) -> nan
19362-
// maximum(X, nan) -> nan
19363-
if (AF.isNaN())
19364-
return PropagatesNaN ? N->getOperand(1) : N->getOperand(0);
19361+
// minnum(X, qnan) -> X
19362+
// maxnum(X, qnan) -> X
19363+
// minnum(X, snan) -> qnan
19364+
// maxnum(X, snan) -> qnan
19365+
// minimum(X, nan) -> qnan
19366+
// maximum(X, nan) -> qnan
19367+
// minimumnum(X, nan) -> X
19368+
// maximumnum(X, nan) -> X
19369+
if (AF.isNaN()) {
19370+
if (PropAllNaNsToQNaNs || (AF.isSignaling() && PropOnlySNaNsToQNaNs)) {
19371+
if (AF.isSignaling())
19372+
return DAG.getConstantFP(AF.makeQuiet(), SDLoc(N), VT);
19373+
return N->getOperand(1);
19374+
}
19375+
return N->getOperand(0);
19376+
}
1936519377

1936619378
// In the following folds, inf can be replaced with the largest finite
1936719379
// float, if the ninf flag is set.
1936819380
if (AF.isInfinity() || (Flags.hasNoInfs() && AF.isLargest())) {
19369-
// minnum(X, -inf) -> -inf
19370-
// maxnum(X, +inf) -> +inf
19381+
// minnum(X, -inf) -> -inf (ignoring sNaN -> qNaN propagation)
19382+
// maxnum(X, +inf) -> +inf (ignoring sNaN -> qNaN propagation)
1937119383
// minimum(X, -inf) -> -inf if nnan
1937219384
// maximum(X, +inf) -> +inf if nnan
19373-
if (IsMin == AF.isNegative() && (!PropagatesNaN || Flags.hasNoNaNs()))
19385+
// minimumnum(X, -inf) -> -inf
19386+
// maximumnum(X, +inf) -> +inf
19387+
if (IsMin == AF.isNegative() &&
19388+
(!PropAllNaNsToQNaNs || Flags.hasNoNaNs()))
1937419389
return N->getOperand(1);
1937519390

1937619391
// minnum(X, +inf) -> X if nnan
1937719392
// maxnum(X, -inf) -> X if nnan
19378-
// minimum(X, +inf) -> X
19379-
// maximum(X, -inf) -> X
19380-
if (IsMin != AF.isNegative() && (PropagatesNaN || Flags.hasNoNaNs()))
19393+
// minimum(X, +inf) -> X (ignoring quieting of sNaNs)
19394+
// maximum(X, -inf) -> X (ignoring quieting of sNaNs)
19395+
// minimumnum(X, +inf) -> X if nnan
19396+
// maximumnum(X, -inf) -> X if nnan
19397+
if (IsMin != AF.isNegative() && (PropAllNaNsToQNaNs || Flags.hasNoNaNs()))
1938119398
return N->getOperand(0);
1938219399
}
1938319400
}
1938419401

19402+
// There are no VECREDUCE variants of FMINIMUMNUM or FMAXIMUMNUM
19403+
if (Opc == ISD::FMINIMUMNUM || Opc == ISD::FMAXIMUMNUM)
19404+
return SDValue();
19405+
1938519406
if (SDValue SD = reassociateReduction(
19386-
PropagatesNaN
19407+
PropAllNaNsToQNaNs
1938719408
? (IsMin ? ISD::VECREDUCE_FMINIMUM : ISD::VECREDUCE_FMAXIMUM)
1938819409
: (IsMin ? ISD::VECREDUCE_FMIN : ISD::VECREDUCE_FMAX),
1938919410
Opc, SDLoc(N), VT, N0, N1, Flags))

‎llvm/test/CodeGen/X86/fmaxnum.ll‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -645,11 +645,47 @@ define float @test_maxnum_const_op2(float %x) {
645645
ret float %r
646646
}
647647

648-
define float @test_maxnum_const_nan(float %x) {
649-
; CHECK-LABEL: test_maxnum_const_nan:
650-
; CHECK: # %bb.0:
651-
; CHECK-NEXT: retq
652-
%r = call float @llvm.maxnum.f32(float %x, float 0x7fff000000000000)
648+
define float @test_maxnum_const_nan(float %x, float %y) {
649+
; SSE-LABEL: test_maxnum_const_nan:
650+
; SSE: # %bb.0:
651+
; SSE-NEXT: movaps %xmm1, %xmm0
652+
; SSE-NEXT: retq
653+
;
654+
; AVX-LABEL: test_maxnum_const_nan:
655+
; AVX: # %bb.0:
656+
; AVX-NEXT: vmovaps %xmm1, %xmm0
657+
; AVX-NEXT: retq
658+
%r = call float @llvm.maxnum.f32(float %y, float 0x7fff000000000000)
659+
ret float %r
660+
}
661+
662+
; nnan maxnum(Y, -inf) -> Y
663+
define float @test_maxnum_neg_inf_nnan(float %x, float %y) nounwind {
664+
; SSE-LABEL: test_maxnum_neg_inf_nnan:
665+
; SSE: # %bb.0:
666+
; SSE-NEXT: movaps %xmm1, %xmm0
667+
; SSE-NEXT: retq
668+
;
669+
; AVX-LABEL: test_maxnum_neg_inf_nnan:
670+
; AVX: # %bb.0:
671+
; AVX-NEXT: vmovaps %xmm1, %xmm0
672+
; AVX-NEXT: retq
673+
%r = call nnan float @llvm.maxnum.f32(float %y, float 0xfff0000000000000)
674+
ret float %r
675+
}
676+
677+
; Test SNaN quieting
678+
define float @test_maxnum_snan(float %x) {
679+
; SSE-LABEL: test_maxnum_snan:
680+
; SSE: # %bb.0:
681+
; SSE-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
682+
; SSE-NEXT: retq
683+
;
684+
; AVX-LABEL: test_maxnum_snan:
685+
; AVX: # %bb.0:
686+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
687+
; AVX-NEXT: retq
688+
%r = call float @llvm.maxnum.f32(float 0x7ff4000000000000, float %x)
653689
ret float %r
654690
}
655691

‎llvm/test/CodeGen/X86/fminimum-fmaximum.ll‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2649,3 +2649,102 @@ define <4 x bfloat> @test_fmaximum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
26492649
%r = call <4 x bfloat> @llvm.maximum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
26502650
ret <4 x bfloat> %r
26512651
}
2652+
2653+
; nnan minimum(Y, +inf) -> Y
2654+
define float @test_fminimum_inf_nnan(float %x, float %y) nounwind {
2655+
; SSE2-LABEL: test_fminimum_inf_nnan:
2656+
; SSE2: # %bb.0:
2657+
; SSE2-NEXT: movaps %xmm1, %xmm0
2658+
; SSE2-NEXT: retq
2659+
;
2660+
; AVX-LABEL: test_fminimum_inf_nnan:
2661+
; AVX: # %bb.0:
2662+
; AVX-NEXT: vmovaps %xmm1, %xmm0
2663+
; AVX-NEXT: retq
2664+
;
2665+
; AVX10_2-LABEL: test_fminimum_inf_nnan:
2666+
; AVX10_2: # %bb.0:
2667+
; AVX10_2-NEXT: vmovaps %xmm1, %xmm0
2668+
; AVX10_2-NEXT: retq
2669+
;
2670+
; X86-LABEL: test_fminimum_inf_nnan:
2671+
; X86: # %bb.0:
2672+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2673+
; X86-NEXT: retl
2674+
%1 = call nnan float @llvm.minimum.f32(float %y, float 0x7ff0000000000000)
2675+
ret float %1
2676+
}
2677+
2678+
; nnan maximum(Y, -inf) -> Y
2679+
define float @test_fmaximum_neg_inf_nnan(float %x, float %y) nounwind {
2680+
; SSE2-LABEL: test_fmaximum_neg_inf_nnan:
2681+
; SSE2: # %bb.0:
2682+
; SSE2-NEXT: movaps %xmm1, %xmm0
2683+
; SSE2-NEXT: retq
2684+
;
2685+
; AVX-LABEL: test_fmaximum_neg_inf_nnan:
2686+
; AVX: # %bb.0:
2687+
; AVX-NEXT: vmovaps %xmm1, %xmm0
2688+
; AVX-NEXT: retq
2689+
;
2690+
; AVX10_2-LABEL: test_fmaximum_neg_inf_nnan:
2691+
; AVX10_2: # %bb.0:
2692+
; AVX10_2-NEXT: vmovaps %xmm1, %xmm0
2693+
; AVX10_2-NEXT: retq
2694+
;
2695+
; X86-LABEL: test_fmaximum_neg_inf_nnan:
2696+
; X86: # %bb.0:
2697+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2698+
; X86-NEXT: retl
2699+
%1 = call nnan float @llvm.maximum.f32(float %y, float 0xfff0000000000000)
2700+
ret float %1
2701+
}
2702+
2703+
; Test SNaN quieting
2704+
define float @test_fmaximum_snan(float %x) {
2705+
; SSE2-LABEL: test_fmaximum_snan:
2706+
; SSE2: # %bb.0:
2707+
; SSE2-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2708+
; SSE2-NEXT: retq
2709+
;
2710+
; AVX-LABEL: test_fmaximum_snan:
2711+
; AVX: # %bb.0:
2712+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2713+
; AVX-NEXT: retq
2714+
;
2715+
; AVX10_2-LABEL: test_fmaximum_snan:
2716+
; AVX10_2: # %bb.0:
2717+
; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2718+
; AVX10_2-NEXT: retq
2719+
;
2720+
; X86-LABEL: test_fmaximum_snan:
2721+
; X86: # %bb.0:
2722+
; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
2723+
; X86-NEXT: retl
2724+
%1 = tail call float @llvm.maximum.f32(float 0x7ff4000000000000, float %x)
2725+
ret float %1
2726+
}
2727+
2728+
define float @test_fminimum_snan(float %x) {
2729+
; SSE2-LABEL: test_fminimum_snan:
2730+
; SSE2: # %bb.0:
2731+
; SSE2-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2732+
; SSE2-NEXT: retq
2733+
;
2734+
; AVX-LABEL: test_fminimum_snan:
2735+
; AVX: # %bb.0:
2736+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2737+
; AVX-NEXT: retq
2738+
;
2739+
; AVX10_2-LABEL: test_fminimum_snan:
2740+
; AVX10_2: # %bb.0:
2741+
; AVX10_2-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
2742+
; AVX10_2-NEXT: retq
2743+
;
2744+
; X86-LABEL: test_fminimum_snan:
2745+
; X86: # %bb.0:
2746+
; X86-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
2747+
; X86-NEXT: retl
2748+
%1 = tail call float @llvm.minimum.f32(float 0x7ff4000000000000, float %x)
2749+
ret float %1
2750+
}

‎llvm/test/CodeGen/X86/fminimumnum-fmaximumnum.ll‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2479,3 +2479,102 @@ define <4 x bfloat> @test_fmaximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) n
24792479
%r = call <4 x bfloat> @llvm.maximumnum.v4bf16(<4 x bfloat> %x, <4 x bfloat> %y)
24802480
ret <4 x bfloat> %r
24812481
}
2482+
2483+
; nnan minimumnum(Y, +inf) -> Y
2484+
define float @test_fminimumnum_inf_nnan(float %x, float %y) nounwind {
2485+
; SSE2-LABEL: test_fminimumnum_inf_nnan:
2486+
; SSE2: # %bb.0:
2487+
; SSE2-NEXT: movaps %xmm1, %xmm0
2488+
; SSE2-NEXT: retq
2489+
;
2490+
; AVX-LABEL: test_fminimumnum_inf_nnan:
2491+
; AVX: # %bb.0:
2492+
; AVX-NEXT: vmovaps %xmm1, %xmm0
2493+
; AVX-NEXT: retq
2494+
;
2495+
; AVX10_2-LABEL: test_fminimumnum_inf_nnan:
2496+
; AVX10_2: # %bb.0:
2497+
; AVX10_2-NEXT: vmovaps %xmm1, %xmm0
2498+
; AVX10_2-NEXT: retq
2499+
;
2500+
; X86-LABEL: test_fminimumnum_inf_nnan:
2501+
; X86: # %bb.0:
2502+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2503+
; X86-NEXT: retl
2504+
%1 = call nnan float @llvm.minimumnum.f32(float %y, float 0x7ff0000000000000)
2505+
ret float %1
2506+
}
2507+
2508+
; nnan maximumnum(Y, -inf) -> Y
2509+
define float @test_fmaximumnum_neg_inf_nnan(float %x, float %y) nounwind {
2510+
; SSE2-LABEL: test_fmaximumnum_neg_inf_nnan:
2511+
; SSE2: # %bb.0:
2512+
; SSE2-NEXT: movaps %xmm1, %xmm0
2513+
; SSE2-NEXT: retq
2514+
;
2515+
; AVX-LABEL: test_fmaximumnum_neg_inf_nnan:
2516+
; AVX: # %bb.0:
2517+
; AVX-NEXT: vmovaps %xmm1, %xmm0
2518+
; AVX-NEXT: retq
2519+
;
2520+
; AVX10_2-LABEL: test_fmaximumnum_neg_inf_nnan:
2521+
; AVX10_2: # %bb.0:
2522+
; AVX10_2-NEXT: vmovaps %xmm1, %xmm0
2523+
; AVX10_2-NEXT: retq
2524+
;
2525+
; X86-LABEL: test_fmaximumnum_neg_inf_nnan:
2526+
; X86: # %bb.0:
2527+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2528+
; X86-NEXT: retl
2529+
%1 = call nnan float @llvm.maximumnum.f32(float %y, float 0xfff0000000000000)
2530+
ret float %1
2531+
}
2532+
2533+
; Test we propagate the non-NaN arg, even if one arg is SNaN
2534+
define float @test_fmaximumnum_snan(float %x, float %y) {
2535+
; SSE2-LABEL: test_fmaximumnum_snan:
2536+
; SSE2: # %bb.0:
2537+
; SSE2-NEXT: movaps %xmm1, %xmm0
2538+
; SSE2-NEXT: retq
2539+
;
2540+
; AVX-LABEL: test_fmaximumnum_snan:
2541+
; AVX: # %bb.0:
2542+
; AVX-NEXT: vmovaps %xmm1, %xmm0
2543+
; AVX-NEXT: retq
2544+
;
2545+
; AVX10_2-LABEL: test_fmaximumnum_snan:
2546+
; AVX10_2: # %bb.0:
2547+
; AVX10_2-NEXT: vmovaps %xmm1, %xmm0
2548+
; AVX10_2-NEXT: retq
2549+
;
2550+
; X86-LABEL: test_fmaximumnum_snan:
2551+
; X86: # %bb.0:
2552+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2553+
; X86-NEXT: retl
2554+
%1 = tail call float @llvm.maximumnum.f32(float 0x7ff4000000000000, float %y)
2555+
ret float %1
2556+
}
2557+
2558+
define float @test_fminimumnum_snan(float %x, float %y) {
2559+
; SSE2-LABEL: test_fminimumnum_snan:
2560+
; SSE2: # %bb.0:
2561+
; SSE2-NEXT: movaps %xmm1, %xmm0
2562+
; SSE2-NEXT: retq
2563+
;
2564+
; AVX-LABEL: test_fminimumnum_snan:
2565+
; AVX: # %bb.0:
2566+
; AVX-NEXT: vmovaps %xmm1, %xmm0
2567+
; AVX-NEXT: retq
2568+
;
2569+
; AVX10_2-LABEL: test_fminimumnum_snan:
2570+
; AVX10_2: # %bb.0:
2571+
; AVX10_2-NEXT: vmovaps %xmm1, %xmm0
2572+
; AVX10_2-NEXT: retq
2573+
;
2574+
; X86-LABEL: test_fminimumnum_snan:
2575+
; X86: # %bb.0:
2576+
; X86-NEXT: flds {{[0-9]+}}(%esp)
2577+
; X86-NEXT: retl
2578+
%1 = tail call float @llvm.minimumnum.f32(float 0x7ff4000000000000, float %y)
2579+
ret float %1
2580+
}

‎llvm/test/CodeGen/X86/fminnum.ll‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -645,11 +645,47 @@ define float @test_minnum_const_op2(float %x) {
645645
ret float %r
646646
}
647647

648-
define float @test_minnum_const_nan(float %x) {
649-
; CHECK-LABEL: test_minnum_const_nan:
650-
; CHECK: # %bb.0:
651-
; CHECK-NEXT: retq
652-
%r = call float @llvm.minnum.f32(float %x, float 0x7fff000000000000)
648+
define float @test_minnum_const_nan(float %x, float %y) {
649+
; SSE-LABEL: test_minnum_const_nan:
650+
; SSE: # %bb.0:
651+
; SSE-NEXT: movaps %xmm1, %xmm0
652+
; SSE-NEXT: retq
653+
;
654+
; AVX-LABEL: test_minnum_const_nan:
655+
; AVX: # %bb.0:
656+
; AVX-NEXT: vmovaps %xmm1, %xmm0
657+
; AVX-NEXT: retq
658+
%r = call float @llvm.minnum.f32(float %y, float 0x7fff000000000000)
659+
ret float %r
660+
}
661+
662+
; nnan minnum(Y, +inf) -> Y
663+
define float @test_minnum_inf_nnan(float %x, float %y) nounwind {
664+
; SSE-LABEL: test_minnum_inf_nnan:
665+
; SSE: # %bb.0:
666+
; SSE-NEXT: movaps %xmm1, %xmm0
667+
; SSE-NEXT: retq
668+
;
669+
; AVX-LABEL: test_minnum_inf_nnan:
670+
; AVX: # %bb.0:
671+
; AVX-NEXT: vmovaps %xmm1, %xmm0
672+
; AVX-NEXT: retq
673+
%r = call nnan float @llvm.minnum.f32(float %y, float 0x7ff0000000000000)
674+
ret float %r
675+
}
676+
677+
; Test SNaN quieting
678+
define float @test_minnum_snan(float %x) {
679+
; SSE-LABEL: test_minnum_snan:
680+
; SSE: # %bb.0:
681+
; SSE-NEXT: movss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
682+
; SSE-NEXT: retq
683+
;
684+
; AVX-LABEL: test_minnum_snan:
685+
; AVX: # %bb.0:
686+
; AVX-NEXT: vmovss {{.*#+}} xmm0 = [NaN,0.0E+0,0.0E+0,0.0E+0]
687+
; AVX-NEXT: retq
688+
%r = call float @llvm.minnum.f32(float 0x7ff4000000000000, float %x)
653689
ret float %r
654690
}
655691

0 commit comments

Comments
 (0)