[InstCombine] Inverse is.fpclass mask operand, when profitable

The `@llvm.is.fpclass` intrinsic is matched and generated by the InstCombine
pass. When the number of set mask bits is greater than the number of the unset
bits, it's profitable to replace the `is.fpclass(x, mask)` intrinsic call with
a sequence of `!is.fpclass(x, ~mask)` operations.

The following IR snippets are semantically equivalent:
```LLVM
define i1 @src(float %src) {
  ; 639 == 0b1001111111
  %class = call i1 @llvm.is.fpclass.f32(float %src, i32 639)
  ret i1 %class
}

define i1 @tgt(float %src) {
  ; 384 == 0b0110000000 == ~0b1001111111 & 0x3ff == ~639 & 0x3ff
  %class = call i1 @llvm.is.fpclass.f32(float %src, i32 384)
  %not = xor i1 %class, true
  ret i1 %not
}
```

However, the generated code is more efficient for the 2nd IR sequence,
at least on some targets.

References:
* https://alive2.llvm.org/ce/z/kkZhDt
* https://godbolt.org/z/5WE8Wb3vz

Author: vmustya

llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp

@@ -1522,9 +1522,21 @@ Value *InstCombinerImpl::foldLogicOfFCmps(FCmpInst *LHS, FCmpInst *RHS,
       if (ClassValLHS == ClassValRHS) {
         unsigned CombinedMask = IsAnd ? (ClassMaskLHS & ClassMaskRHS)
                                       : (ClassMaskLHS | ClassMaskRHS);
-        return Builder.CreateIntrinsic(
-            Intrinsic::is_fpclass, {ClassValLHS->getType()},
-            {ClassValLHS, Builder.getInt32(CombinedMask)});
+        unsigned InverseCombinedMask = ~CombinedMask & fcAllFlags;
+
+        // If the number of bits set in the combined mask is greater than the
+        // number of the unset bits, it is more efficient to use the inverse
+        // mask and invert the result.
+        bool IsInverse = popcount(CombinedMask) > popcount(InverseCombinedMask);
+        auto *MaskVal =
+            Builder.getInt32(IsInverse ? InverseCombinedMask : CombinedMask);
+
+        auto *II = Builder.CreateIntrinsic(Intrinsic::is_fpclass,
+                                           {ClassValLHS->getType()},
+                                           {ClassValLHS, MaskVal});
+        if (IsInverse)
+          return Builder.CreateNot(II);
+        return II;
       }
     }
   }
@@ -1610,10 +1622,25 @@ Instruction *InstCombinerImpl::foldLogicOfIsFPClass(BinaryOperator &BO,
   bool IsRHSClass =
       match(Op1, m_OneUse(m_Intrinsic<Intrinsic::is_fpclass>(
                      m_Value(ClassVal1), m_ConstantInt(ClassMask1))));
-  if ((((IsLHSClass || matchIsFPClassLikeFCmp(Op0, ClassVal0, ClassMask0)) &&
-        (IsRHSClass || matchIsFPClassLikeFCmp(Op1, ClassVal1, ClassMask1)))) &&
+
+  bool IsLHSInverseClass =
+      match(Op0, m_OneUse(m_Not(m_OneUse(m_Intrinsic<Intrinsic::is_fpclass>(
+                     m_Value(ClassVal0), m_ConstantInt(ClassMask0))))));
+  bool IsRHSInverseClass =
+      match(Op1, m_OneUse(m_Not(m_OneUse(m_Intrinsic<Intrinsic::is_fpclass>(
+                     m_Value(ClassVal1), m_ConstantInt(ClassMask1))))));
+
+  if ((((IsLHSClass || IsLHSInverseClass ||
+         matchIsFPClassLikeFCmp(Op0, ClassVal0, ClassMask0)) &&
+        (IsRHSClass || IsRHSInverseClass ||
+         matchIsFPClassLikeFCmp(Op1, ClassVal1, ClassMask1)))) &&
       ClassVal0 == ClassVal1) {
     unsigned NewClassMask;
+    if (IsLHSInverseClass)
+      ClassMask0 = ~ClassMask0 & fcAllFlags;
+    if (IsRHSInverseClass)
+      ClassMask1 = ~ClassMask1 & fcAllFlags;
+
     switch (BO.getOpcode()) {
     case Instruction::And:
       NewClassMask = ClassMask0 & ClassMask1;
@@ -4651,10 +4678,17 @@ Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) {
 
     if (II->getIntrinsicID() == Intrinsic::is_fpclass) {
       ConstantInt *ClassMask = cast<ConstantInt>(II->getArgOperand(1));
-      II->setArgOperand(
-          1, ConstantInt::get(ClassMask->getType(),
-                              ~ClassMask->getZExtValue() & fcAllFlags));
-      return replaceInstUsesWith(I, II);
+      auto ClassMaskValue = ClassMask->getZExtValue();
+      auto InverseMaskValue = ~ClassMaskValue & fcAllFlags;
+
+      // If the number of set bits in the class mask is less than the number of
+      // set bits in the inverse mask, it's more efficient to keep the "not"
+      // instruction instead of inverting the class mask.
+      if (popcount(ClassMaskValue) > popcount(InverseMaskValue)) {
+        II->setArgOperand(
+            1, ConstantInt::get(ClassMask->getType(), InverseMaskValue));
+        return replaceInstUsesWith(I, II);
+      }
     }
   }
 

llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp

@@ -1050,6 +1050,20 @@ Instruction *InstCombinerImpl::foldIntrinsicIsFPClass(IntrinsicInst &II) {
   if (Mask == Known.KnownFPClasses)
     return replaceInstUsesWith(II, ConstantInt::get(II.getType(), true));
 
+  // If the number of set bits in the mask is greater than the number of the
+  // unset bits, it's more efficient to inverse the mask and the intrinsic
+  // result:
+  // if.fpclass(x, mask) -> !if.fpclass(x, ~mask)
+  //
+  auto InverseMask = ~Mask & fcAllFlags;
+  if (popcount<unsigned>(Mask) > popcount<unsigned>(InverseMask)) {
+    auto *NewII =
+        Builder.CreateIntrinsic(Intrinsic::is_fpclass, {Src0->getType()},
+                                {Src0, Builder.getInt32(InverseMask)});
+    auto *Not = Builder.CreateNot(NewII);
+    return replaceInstUsesWith(II, Not);
+  }
+
   return nullptr;
 }
 

llvm/test/Transforms/InstCombine/combine-is.fpclass-and-fcmp.ll

@@ -189,7 +189,8 @@ define i1 @fcmp_issubnormal_and_class_finite(half %x) {
 
 define i1 @class_inf_or_fcmp_issubnormal(half %x) {
 ; CHECK-LABEL: @class_inf_or_fcmp_issubnormal(
-; CHECK-NEXT:    [[OR:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 756)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 267)
+; CHECK-NEXT:    [[OR:%.*]] = xor i1 [[TMP1]], true
 ; CHECK-NEXT:    ret i1 [[OR]]
 ;
   %fabs = call half @llvm.fabs.f16(half %x)
@@ -289,7 +290,8 @@ define i1 @class_normal_or_fcmp_oeq_zero(half %x) {
 
 define i1 @fcmp_ueq_zero_or_class_normal(half %x) {
 ; CHECK-LABEL: @fcmp_ueq_zero_or_class_normal(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 363)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 660)
+; CHECK-NEXT:    [[CLASS:%.*]] = xor i1 [[TMP1]], true
 ; CHECK-NEXT:    ret i1 [[CLASS]]
 ;
   %ueq.inf = fcmp ueq half %x, 0.0
@@ -300,7 +302,8 @@ define i1 @fcmp_ueq_zero_or_class_normal(half %x) {
 
 define i1 @class_normal_or_fcmp_ueq_zero(half %x) {
 ; CHECK-LABEL: @class_normal_or_fcmp_ueq_zero(
-; CHECK-NEXT:    [[CLASS:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 363)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.is.fpclass.f16(half [[X:%.*]], i32 660)
+; CHECK-NEXT:    [[CLASS:%.*]] = xor i1 [[TMP1]], true
 ; CHECK-NEXT:    ret i1 [[CLASS]]
 ;
   %ueq.inf = fcmp ueq half %x, 0.0