Wrong folding to `fabs` (`NaN` not handled)

[bongjunj]

llvm-project/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp

Lines 2814 to 2885 in 1ceccbb

	// Canonicalize select with fcmp to fabs(). -0.0 makes this tricky. We need
	// fast-math-flags (nsz) or fsub with +0.0 (not fneg) for this to work.
	static Instruction *foldSelectWithFCmpToFabs(SelectInst &SI,
	InstCombinerImpl &IC) {
	Value *CondVal = SI.getCondition();
	bool ChangedFMF = false;
	for (bool Swap : {false, true}) {
	Value *TrueVal = SI.getTrueValue();
	Value *X = SI.getFalseValue();
	CmpInst::Predicate Pred;
	if (Swap)
	std::swap(TrueVal, X);
	if (!match(CondVal, m_FCmp(Pred, m_Specific(X), m_AnyZeroFP())))
	continue;
	// fold (X <= +/-0.0) ? (0.0 - X) : X to fabs(X), when 'Swap' is false
	// fold (X > +/-0.0) ? X : (0.0 - X) to fabs(X), when 'Swap' is true
	if (match(TrueVal, m_FSub(m_PosZeroFP(), m_Specific(X)))) {
	if (!Swap && (Pred == FCmpInst::FCMP_OLE || Pred == FCmpInst::FCMP_ULE)) {
	Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
	return IC.replaceInstUsesWith(SI, Fabs);
	}
	if (Swap && (Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_UGT)) {
	Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
	return IC.replaceInstUsesWith(SI, Fabs);
	}
	}
	if (!match(TrueVal, m_FNeg(m_Specific(X))))
	return nullptr;
	// Forward-propagate nnan and ninf from the fneg to the select.
	// If all inputs are not those values, then the select is not either.
	// Note: nsz is defined differently, so it may not be correct to propagate.
	FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags();
	if (FMF.noNaNs() && !SI.hasNoNaNs()) {
	SI.setHasNoNaNs(true);
	ChangedFMF = true;
	}
	if (FMF.noInfs() && !SI.hasNoInfs()) {
	SI.setHasNoInfs(true);
	ChangedFMF = true;
	}
	// With nsz, when 'Swap' is false:
	// fold (X < +/-0.0) ? -X : X or (X <= +/-0.0) ? -X : X to fabs(X)
	// fold (X > +/-0.0) ? -X : X or (X >= +/-0.0) ? -X : X to -fabs(x)
	// when 'Swap' is true:
	// fold (X > +/-0.0) ? X : -X or (X >= +/-0.0) ? X : -X to fabs(X)
	// fold (X < +/-0.0) ? X : -X or (X <= +/-0.0) ? X : -X to -fabs(X)
	//
	// Note: We require "nnan" for this fold because fcmp ignores the signbit
	// of NAN, but IEEE-754 specifies the signbit of NAN values with
	// fneg/fabs operations.
	if (!SI.hasNoSignedZeros() || !SI.hasNoNaNs())
	return nullptr;
	if (Swap)
	Pred = FCmpInst::getSwappedPredicate(Pred);
	bool IsLTOrLE = Pred == FCmpInst::FCMP_OLT || Pred == FCmpInst::FCMP_OLE ||
	Pred == FCmpInst::FCMP_ULT || Pred == FCmpInst::FCMP_ULE;
	bool IsGTOrGE = Pred == FCmpInst::FCMP_OGT || Pred == FCmpInst::FCMP_OGE ||
	Pred == FCmpInst::FCMP_UGT || Pred == FCmpInst::FCMP_UGE;
	if (IsLTOrLE) {
	Value *Fabs = IC.Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, &SI);
	return IC.replaceInstUsesWith(SI, Fabs);
	}

Alive2 report: https://alive2.llvm.org/ce/z/y2UAwe

----------------------------------------
define double @select_fcmp_ole_zero.2(double %x) {
#0:
  %lezero = fcmp ole double %x, 0.000000
  %negx = fsub double 0.000000, %x
  %fabs = select i1 %lezero, double %negx, double %x
  ret double %fabs
}
=>
define double @select_fcmp_ole_zero.2(double %x) {
#0:
  %fabs = fabs double %x
  ret double %fabs
}
Transformation doesn't verify!

ERROR: Value mismatch

Example:
double %x = #xfff0000000000008 (SNaN)

Source:
i1 %lezero = #x0 (0)
double %negx = #x7ff0000000000008 (SNaN)
double %fabs = #xfff0000000000008 (SNaN)

Target:
double %fabs = #x7ff0000000000008 (SNaN)
Source value: #xfff0000000000008 (SNaN)
Target value: #x7ff0000000000008 (SNaN)


----------------------------------------
define double @select_fcmp_nnan_ole_zero.2(double %x) {
#0:
  %lezero = fcmp ole double %x, 0.000000
  %negx = fsub nnan double 0.000000, %x
  %fabs = select i1 %lezero, double %negx, double %x
  ret double %fabs
}
=>
define double @select_fcmp_nnan_ole_zero.2(double %x) {
#0:
  %fabs = fabs double %x
  ret double %fabs
}
Transformation doesn't verify!

ERROR: Value mismatch

Example:
double %x = #xfff0000000004000 (SNaN)

Source:
i1 %lezero = #x0 (0)
double %negx = poison
double %fabs = #xfff0000000004000 (SNaN)

Target:
double %fabs = #x7ff0000000004000 (SNaN)
Source value: #xfff0000000004000 (SNaN)
Target value: #x7ff0000000004000 (SNaN)


----------------------------------------
define half @select_fcmp_nnan_ugt_negzero.2(half %x) {
#0:
  %#1 = fcmp ugt half %x, 0x8000
  %negx = fsub nnan half 0x0000, %x
  %#2 = fadd half %negx, 0x0000
  %fabs = select i1 %#1, half %x, half %#2
  ret half %fabs
}
=>
define half @select_fcmp_nnan_ugt_negzero.2(half %x) {
#0:
  %fabs = fabs half %x
  ret half %fabs
}
Transformation doesn't verify!

ERROR: Value mismatch

Example:
half %x = #xfc04 (SNaN)

Source:
i1 %#1 = #x1 (1)
half %negx = poison
half %#2 = poison
half %fabs = #xfc04 (SNaN)

Target:
half %fabs = #x7c04 (SNaN)
Source value: #xfc04 (SNaN)
Target value: #x7c04 (SNaN)

Summary:
  0 correct transformations
  3 incorrect transformations
  0 failed-to-prove transformations
  0 Alive2 errors

[dtcxzyw]

Fixed by #121580