Avoid optimization for maxnum(x, <sNaN, qNaN>)

Instead of transforming maxnum(x, <sNaN, qNaN>) -> x,
avoid folding the maxnum away so the sNaN element can
correctly be propagated as a qNaN.

Author: LewisCrawford

llvm/lib/Analysis/InstructionSimplify.cpp

@@ -6742,14 +6742,22 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
 
     // minnum(x, qnan) -> x
     // maxnum(x, qnan) -> x
-    // minnum(x, snan) -> qnan (or x for vectors mixing snans and qnans)
-    // maxnum(x, snan) -> qnan (or x for vectors mixing snans and qnans)
+    // minnum(x, snan) -> qnan
+    // maxnum(x, snan) -> qnan
     // minimum(X, nan) -> qnan
     // maximum(X, nan) -> qnan
-    if (PropagateSNaN && match(Op1, m_sNaN()))
+    if (PropagateSNaN && match(Op1, m_sNaN())) {
       return propagateNaN(cast<Constant>(Op1));
-    else if (match(Op1, m_NaN()))
-      return PropagateNaN ? propagateNaN(cast<Constant>(Op1)) : Op0;
+    } else if (match(Op1, m_NaN())) {
+      if (PropagateNaN)
+        return propagateNaN(cast<Constant>(Op1));
+      // In cases like mixed <sNaN, qNaN> vectors, avoid the optimization to
+      // allow correct sNaN propagation where necessary.
+      else if (PropagateSNaN && !match(Op1, m_qNaN()))
+        break;
+      else
+        return Op0;
+    }
 
     // In the following folds, inf can be replaced with the largest finite
     // float, if the ninf flag is set.

llvm/test/Transforms/InstSimplify/fminmax-folds.ll

@@ -1355,31 +1355,35 @@ define <2 x double> @minnum_snan_op1_vec(<2 x double> %x) {
 
 define <2 x double> @maxnum_snan_qnan_op0_vec(<2 x double> %x) {
 ; CHECK-LABEL: @maxnum_snan_qnan_op0_vec(
-; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> <double 0x7FF4000000000000, double 0x7FFF000000000000>, <2 x double> [[X1:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[X]]
 ;
   %r = call <2 x double> @llvm.maxnum.v2f64(<2 x double> <double 0x7ff4000000000000, double 0x7fff000000000000>, <2 x double> %x)
   ret <2 x double> %r
 }
 
 define <2 x double> @maxnum_snan_qnan_op1_vec(<2 x double> %x) {
 ; CHECK-LABEL: @maxnum_snan_qnan_op1_vec(
-; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call <2 x double> @llvm.maxnum.v2f64(<2 x double> [[X1:%.*]], <2 x double> <double 0x7FF400000000DEAD, double 0x7FFF000000000000>)
+; CHECK-NEXT:    ret <2 x double> [[X]]
 ;
   %r = call <2 x double> @llvm.maxnum.v2f64(<2 x double> %x, <2 x double> <double 0x7ff400000000dead, double 0x7fff000000000000>)
   ret <2 x double> %r
 }
 
 define <2 x double> @minnum_snan_qnan_op0_vec(<2 x double> %x) {
 ; CHECK-LABEL: @minnum_snan_qnan_op0_vec(
-; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> <double 0x7FF4000DEAD00000, double 0x7FFF000000000000>, <2 x double> [[X1:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[X]]
 ;
   %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> <double 0x7ff4000dead00000, double 0x7fff000000000000>, <2 x double> %x)
   ret <2 x double> %r
 }
 
 define <2 x double> @minnum_snan_qnan_op1_vec(<2 x double> %x) {
 ; CHECK-LABEL: @minnum_snan_qnan_op1_vec(
-; CHECK-NEXT:    ret <2 x double> [[X:%.*]]
+; CHECK-NEXT:    [[X:%.*]] = call <2 x double> @llvm.minnum.v2f64(<2 x double> [[X1:%.*]], <2 x double> <double 0x7FF400DEAD00DEAD, double 0x7FFF00DEAD00DEAD>)
+; CHECK-NEXT:    ret <2 x double> [[X]]
 ;
   %r = call <2 x double> @llvm.minnum.v2f64(<2 x double> %x, <2 x double> <double 0x7ff400dead00dead, double 0x7fff00dead00dead>)
   ret <2 x double> %r