[GlobalISel] Introduce G_POISON

Author: mtsokol

llvm/docs/GlobalISel/GenericOpcode.rst

@@ -24,6 +24,15 @@ An undefined value.
 
   %0:_(s32) = G_IMPLICIT_DEF
 
+G_POISON
+^^^^^^^^
+
+A poison value.
+
+.. code-block:: none
+
+  %0:_(s32) = G_POISON
+
 G_CONSTANT
 ^^^^^^^^^^
 

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

@@ -431,16 +431,27 @@ class CombinerHelper {
   /// G_IMPLICIT_DEF.
   bool matchAnyExplicitUseIsUndef(MachineInstr &MI) const;
 
+  /// Return true if any explicit use operand on \p MI is defined by a
+  /// G_POISON.
+  bool matchAnyExplicitUseIsPoison(MachineInstr &MI) const;
+
   /// Return true if all register explicit use operands on \p MI are defined by
   /// a G_IMPLICIT_DEF.
   bool matchAllExplicitUsesAreUndef(MachineInstr &MI) const;
 
+  /// Return true if all register explicit use operands on \p MI are defined by
+  /// a G_POISON.
+  bool matchAllExplicitUsesArePoison(MachineInstr &MI) const;
+
   /// Return true if a G_SHUFFLE_VECTOR instruction \p MI has an undef mask.
   bool matchUndefShuffleVectorMask(MachineInstr &MI) const;
 
   /// Return true if a G_STORE instruction \p MI is storing an undef value.
   bool matchUndefStore(MachineInstr &MI) const;
 
+  /// Return true if a G_STORE instruction \p MI is storing a poison value.
+  bool matchPoisonStore(MachineInstr &MI) const;
+
   /// Return true if a G_SELECT instruction \p MI has an undef comparison.
   bool matchUndefSelectCmp(MachineInstr &MI) const;
 
@@ -466,6 +477,9 @@ class CombinerHelper {
   /// Replace an instruction with a G_IMPLICIT_DEF.
   void replaceInstWithUndef(MachineInstr &MI) const;
 
+  /// Replace an instruction with a G_POISON.
+  void replaceInstWithPoison(MachineInstr &MI) const;
+
   /// Delete \p MI and replace all of its uses with its \p OpIdx-th operand.
   void replaceSingleDefInstWithOperand(MachineInstr &MI, unsigned OpIdx) const;
 
@@ -506,6 +520,9 @@ class CombinerHelper {
   /// Check if operand \p OpIdx is undef.
   bool matchOperandIsUndef(MachineInstr &MI, unsigned OpIdx) const;
 
+  /// Check if operand \p OpIdx is poison.
+  bool matchOperandIsPoison(MachineInstr &MI, unsigned OpIdx) const;
+
   /// Check if operand \p OpIdx is known to be a power of 2.
   bool matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI,
                                           unsigned OpIdx) const;

llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h

@@ -343,6 +343,14 @@ class GImplicitDef : public GenericMachineInstr {
   }
 };
 
+/// Represents a G_POISON.
+class GPoison : public GenericMachineInstr {
+public:
+  static bool classof(const MachineInstr *MI) {
+    return MI->getOpcode() == TargetOpcode::G_POISON;
+  }
+};
+
 /// Represents a G_SELECT.
 class GSelect : public GenericMachineInstr {
 public:

llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h

@@ -1017,9 +1017,12 @@ class MachineIRBuilder {
   /// \return a MachineInstrBuilder for the newly created instruction.
   MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index);
 
-  /// Build and insert \p Res = IMPLICIT_DEF.
+  /// Build and insert \p Res = G_IMPLICIT_DEF.
   MachineInstrBuilder buildUndef(const DstOp &Res);
 
+  /// Build and insert \p Res = G_POISON.
+  MachineInstrBuilder buildPoison(const DstOp &Res);
+
   /// Build and insert \p Res = G_MERGE_VALUES \p Op0, ...
   ///
   /// G_MERGE_VALUES combines the input elements contiguously into a larger

llvm/include/llvm/Support/TargetOpcodes.def

@@ -295,9 +295,12 @@ HANDLE_TARGET_OPCODE(G_ABDS)
 /// Generic absolute difference unsigned instruction.
 HANDLE_TARGET_OPCODE(G_ABDU)
 
-
+// Generic implicit definition.
 HANDLE_TARGET_OPCODE(G_IMPLICIT_DEF)
 
+// Generic poison value.
+HANDLE_TARGET_OPCODE(G_POISON)
+
 /// Generic PHI instruction with types.
 HANDLE_TARGET_OPCODE(G_PHI)
 

llvm/include/llvm/Target/GenericOpcodes.td

@@ -92,6 +92,12 @@ def G_IMPLICIT_DEF : GenericInstruction {
   let hasSideEffects = false;
 }
 
+def G_POISON : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins);
+  let hasSideEffects = false;
+}
+
 def G_PHI : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins variable_ops);

llvm/include/llvm/Target/GlobalISel/Combine.td

@@ -430,6 +430,12 @@ def binop_right_undef_to_undef: GICombineRule<
          [{ return Helper.matchOperandIsUndef(*${root}, 2); }]),
   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
 
+def binop_right_poison_to_poison
+    : GICombineRule<(defs root:$root),
+                    (match(wip_match_opcode G_SHL, G_ASHR, G_LSHR):$root,
+                        [{ return Helper.matchOperandIsPoison(*${root}, 2); }]),
+                    (apply [{ Helper.replaceInstWithPoison(*${root}); }])>;
+
 def unary_undef_to_zero: GICombineRule<
   (defs root:$root),
   (match (wip_match_opcode G_ABS):$root,
@@ -447,6 +453,17 @@ def unary_undef_to_undef : GICombineRule<
   (match (unary_undef_to_undef_frags $dst)),
   (apply [{ Helper.replaceInstWithUndef(*${dst}.getParent()); }])>;
 
+def unary_poison_to_poison_frags
+    : GICombinePatFrag<(outs root:$dst), (ins),
+                       !foreach(op,
+                                [G_TRUNC, G_BITCAST, G_ANYEXT, G_PTRTOINT,
+                                 G_INTTOPTR, G_FPTOSI, G_FPTOUI],
+                                (pattern(op $dst, $x), (G_POISON $x)))>;
+def unary_poison_to_poison
+    : GICombineRule<
+          (defs root:$dst), (match(unary_poison_to_poison_frags $dst)),
+          (apply [{ Helper.replaceInstWithPoison(*${dst}.getParent()); }])>;
+
 // Instructions where if any source operand is undef, the instruction can be
 // replaced with undef.
 def propagate_undef_any_op: GICombineRule<
@@ -455,6 +472,15 @@ def propagate_undef_any_op: GICombineRule<
          [{ return Helper.matchAnyExplicitUseIsUndef(*${root}); }]),
   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
 
+// Instructions where if any source operand is poison, the instruction can be
+// replaced with poison.
+def propagate_poison_any_op
+    : GICombineRule<
+          (defs root:$root),
+          (match(wip_match_opcode G_ADD, G_SUB, G_XOR):$root,
+              [{ return Helper.matchAnyExplicitUseIsPoison(*${root}); }]),
+          (apply [{ Helper.replaceInstWithPoison(*${root}); }])>;
+
 // Instructions where if all source operands are undef, the instruction can be
 // replaced with undef.
 def propagate_undef_all_ops: GICombineRule<
@@ -463,6 +489,15 @@ def propagate_undef_all_ops: GICombineRule<
           [{ return Helper.matchAllExplicitUsesAreUndef(*${root}); }]),
   (apply [{ Helper.replaceInstWithUndef(*${root}); }])>;
 
+// Instructions where if all source operands are poison, the instruction can be
+// replaced with poison.
+def propagate_poison_all_ops
+    : GICombineRule<
+          (defs root:$root),
+          (match(wip_match_opcode G_SHUFFLE_VECTOR, G_BUILD_VECTOR):$root,
+              [{ return Helper.matchAllExplicitUsesArePoison(*${root}); }]),
+          (apply [{ Helper.replaceInstWithPoison(*${root}); }])>;
+
 // Replace a G_SHUFFLE_VECTOR with an undef mask with a G_IMPLICIT_DEF.
 def propagate_undef_shuffle_mask: GICombineRule<
   (defs root:$root),
@@ -654,6 +689,13 @@ def erase_undef_store : GICombineRule<
   (apply [{ Helper.eraseInst(*${root}); }])
 >;
 
+// Erase stores of poison values.
+def erase_poison_store
+    : GICombineRule<(defs root:$root),
+                    (match(wip_match_opcode G_STORE):$root,
+                        [{ return Helper.matchPoisonStore(*${root}); }]),
+                    (apply [{ Helper.eraseInst(*${root}); }])>;
+
 def simplify_add_to_sub_matchinfo: GIDefMatchData<"std::tuple<Register, Register>">;
 def simplify_add_to_sub: GICombineRule <
   (defs root:$root, simplify_add_to_sub_matchinfo:$info),
@@ -1955,6 +1997,11 @@ def undef_combines : GICombineGroup<[undef_to_fp_zero, undef_to_int_zero,
                                      erase_undef_store,
                                      insert_extract_vec_elt_out_of_bounds]>;
 
+def poison_combines
+    : GICombineGroup<[binop_right_poison_to_poison, unary_poison_to_poison,
+                      propagate_poison_any_op, propagate_poison_all_ops,
+                      erase_poison_store]>;
+
 def identity_combines : GICombineGroup<[select_same_val, right_identity_zero,
                                         binop_same_val, binop_left_to_zero,
                                         binop_right_to_zero, p2i_to_i2p,
@@ -2006,29 +2053,29 @@ def shuffle_combines : GICombineGroup<[combine_shuffle_concat,
                                        combine_shuffle_undef_rhs,
                                        combine_shuffle_disjoint_mask]>;
 
-def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
-    vector_ops_combines, freeze_combines, cast_combines,
-    insert_vec_elt_combines, extract_vec_elt_combines, combines_for_extload,
-    combine_extracted_vector_load,
-    undef_combines, identity_combines, phi_combines,
-    simplify_add_to_sub, hoist_logic_op_with_same_opcode_hands, shifts_too_big,
-    reassocs, ptr_add_immed_chain, cmp_combines,
-    shl_ashr_to_sext_inreg, sext_inreg_of_load,
-    width_reduction_combines, select_combines,
-    known_bits_simplifications, trunc_shift,
-    not_cmp_fold, opt_brcond_by_inverting_cond,
-    const_combines, xor_of_and_with_same_reg, ptr_add_with_zero,
-    shift_immed_chain, shift_of_shifted_logic_chain, load_or_combine,
-    div_rem_to_divrem, funnel_shift_combines, bitreverse_shift, commute_shift,
-    form_bitfield_extract, constant_fold_binops, constant_fold_fma,
-    constant_fold_cast_op, fabs_fneg_fold,
-    intdiv_combines, mulh_combines, redundant_neg_operands,
-    and_or_disjoint_mask, fma_combines, fold_binop_into_select,
-    sub_add_reg, select_to_minmax,
-    fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
-    simplify_neg_minmax, combine_concat_vector,
-    sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
-    combine_use_vector_truncate, merge_combines, overflow_combines]>;
+def all_combines
+    : GICombineGroup<
+          [integer_reassoc_combines, trivial_combines, vector_ops_combines,
+           freeze_combines, cast_combines, insert_vec_elt_combines,
+           extract_vec_elt_combines, combines_for_extload,
+           combine_extracted_vector_load, undef_combines, poison_combines,
+           identity_combines, phi_combines, simplify_add_to_sub,
+           hoist_logic_op_with_same_opcode_hands, shifts_too_big, reassocs,
+           ptr_add_immed_chain, cmp_combines, shl_ashr_to_sext_inreg,
+           sext_inreg_of_load, width_reduction_combines, select_combines,
+           known_bits_simplifications, trunc_shift, not_cmp_fold,
+           opt_brcond_by_inverting_cond, const_combines,
+           xor_of_and_with_same_reg, ptr_add_with_zero, shift_immed_chain,
+           shift_of_shifted_logic_chain, load_or_combine, div_rem_to_divrem,
+           funnel_shift_combines, bitreverse_shift, commute_shift,
+           form_bitfield_extract, constant_fold_binops, constant_fold_fma,
+           constant_fold_cast_op, fabs_fneg_fold, intdiv_combines,
+           mulh_combines, redundant_neg_operands, and_or_disjoint_mask,
+           fma_combines, fold_binop_into_select, sub_add_reg, select_to_minmax,
+           fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
+           simplify_neg_minmax, combine_concat_vector, sext_trunc, zext_trunc,
+           prefer_sign_combines, shuffle_combines, combine_use_vector_truncate,
+           merge_combines, overflow_combines]>;
 
 // A combine group used to for prelegalizer combiners at -O0. The combines in
 // this group have been selected based on experiments to balance code size and

llvm/lib/CodeGen/GlobalISel/CSEInfo.cpp

@@ -54,6 +54,7 @@ bool CSEConfigFull::shouldCSEOpc(unsigned Opc) {
   case TargetOpcode::G_CONSTANT:
   case TargetOpcode::G_FCONSTANT:
   case TargetOpcode::G_IMPLICIT_DEF:
+  case TargetOpcode::G_POISON:
   case TargetOpcode::G_ZEXT:
   case TargetOpcode::G_SEXT:
   case TargetOpcode::G_ANYEXT:
@@ -82,7 +83,7 @@ bool CSEConfigFull::shouldCSEOpc(unsigned Opc) {
 
 bool CSEConfigConstantOnly::shouldCSEOpc(unsigned Opc) {
   return Opc == TargetOpcode::G_CONSTANT || Opc == TargetOpcode::G_FCONSTANT ||
-         Opc == TargetOpcode::G_IMPLICIT_DEF;
+         Opc == TargetOpcode::G_IMPLICIT_DEF || Opc == TargetOpcode::G_POISON;
 }
 
 std::unique_ptr<CSEConfigBase>

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

@@ -328,6 +328,7 @@ bool CombinerHelper::matchCombineConcatVectors(
       for (const MachineOperand &BuildVecMO : Def->uses())
         Ops.push_back(BuildVecMO.getReg());
       break;
+    case TargetOpcode::G_POISON:
     case TargetOpcode::G_IMPLICIT_DEF: {
       LLT OpType = MRI.getType(Reg);
       // Keep one undef value for all the undef operands.
@@ -2695,13 +2696,26 @@ bool CombinerHelper::matchAnyExplicitUseIsUndef(MachineInstr &MI) const {
   });
 }
 
+bool CombinerHelper::matchAnyExplicitUseIsPoison(MachineInstr &MI) const {
+  return any_of(MI.explicit_uses(), [this](const MachineOperand &MO) {
+    return MO.isReg() && getOpcodeDef(TargetOpcode::G_POISON, MO.getReg(), MRI);
+  });
+}
+
 bool CombinerHelper::matchAllExplicitUsesAreUndef(MachineInstr &MI) const {
   return all_of(MI.explicit_uses(), [this](const MachineOperand &MO) {
     return !MO.isReg() ||
            getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
   });
 }
 
+bool CombinerHelper::matchAllExplicitUsesArePoison(MachineInstr &MI) const {
+  return all_of(MI.explicit_uses(), [this](const MachineOperand &MO) {
+    return !MO.isReg() ||
+           getOpcodeDef(TargetOpcode::G_POISON, MO.getReg(), MRI);
+  });
+}
+
 bool CombinerHelper::matchUndefShuffleVectorMask(MachineInstr &MI) const {
   assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
   ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
@@ -2714,6 +2728,11 @@ bool CombinerHelper::matchUndefStore(MachineInstr &MI) const {
                       MRI);
 }
 
+bool CombinerHelper::matchPoisonStore(MachineInstr &MI) const {
+  assert(MI.getOpcode() == TargetOpcode::G_STORE);
+  return getOpcodeDef(TargetOpcode::G_POISON, MI.getOperand(0).getReg(), MRI);
+}
+
 bool CombinerHelper::matchUndefSelectCmp(MachineInstr &MI) const {
   assert(MI.getOpcode() == TargetOpcode::G_SELECT);
   return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(),
@@ -2953,6 +2972,12 @@ bool CombinerHelper::matchOperandIsUndef(MachineInstr &MI,
          getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
 }
 
+bool CombinerHelper::matchOperandIsPoison(MachineInstr &MI,
+                                          unsigned OpIdx) const {
+  MachineOperand &MO = MI.getOperand(OpIdx);
+  return MO.isReg() && getOpcodeDef(TargetOpcode::G_POISON, MO.getReg(), MRI);
+}
+
 bool CombinerHelper::matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI,
                                                         unsigned OpIdx) const {
   MachineOperand &MO = MI.getOperand(OpIdx);
@@ -2992,6 +3017,12 @@ void CombinerHelper::replaceInstWithUndef(MachineInstr &MI) const {
   MI.eraseFromParent();
 }
 
+void CombinerHelper::replaceInstWithPoison(MachineInstr &MI) const {
+  assert(MI.getNumDefs() == 1 && "Expected only one def?");
+  Builder.buildPoison(MI.getOperand(0));
+  MI.eraseFromParent();
+}
+
 bool CombinerHelper::matchSimplifyAddToSub(
     MachineInstr &MI, std::tuple<Register, Register> &MatchInfo) const {
   Register LHS = MI.getOperand(1).getReg();
@@ -3056,6 +3087,7 @@ bool CombinerHelper::matchCombineInsertVecElts(
   // If we didn't end in a G_IMPLICIT_DEF and the source is not fully
   // overwritten, bail out.
   return TmpInst->getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
+         TmpInst->getOpcode() == TargetOpcode::G_POISON ||
          all_of(MatchInfo, [](Register Reg) { return !!Reg; });
 }
 
@@ -3426,12 +3458,13 @@ bool CombinerHelper::matchUseVectorTruncate(MachineInstr &MI,
   if (I < 2)
     return false;
 
-  // Check the remaining source elements are only G_IMPLICIT_DEF
+  // Check the remaining source elements are only G_IMPLICIT_DEF or G_POISON
   for (; I < NumOperands; ++I) {
     auto SrcMI = MRI.getVRegDef(BuildMI->getSourceReg(I));
     auto SrcMIOpc = SrcMI->getOpcode();
 
-    if (SrcMIOpc != TargetOpcode::G_IMPLICIT_DEF)
+    if (SrcMIOpc != TargetOpcode::G_IMPLICIT_DEF &&
+        SrcMIOpc != TargetOpcode::G_POISON)
       return false;
   }
 
@@ -7892,6 +7925,12 @@ bool CombinerHelper::matchShuffleDisjointMask(MachineInstr &MI,
   if (getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, Shuffle.getSrc2Reg(), MRI))
     return false;
 
+  if (getOpcodeDef(TargetOpcode::G_POISON, Shuffle.getSrc1Reg(), MRI))
+    return false;
+
+  if (getOpcodeDef(TargetOpcode::G_POISON, Shuffle.getSrc2Reg(), MRI))
+    return false;
+
   const LLT DstTy = MRI.getType(Shuffle.getReg(0));
   const LLT Src1Ty = MRI.getType(Shuffle.getSrc1Reg());
   if (!isLegalOrBeforeLegalizer(

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

@@ -3660,6 +3660,8 @@ bool IRTranslator::translate(const Constant &C, Register Reg) {
     EntryBuilder->buildConstant(Reg, *CI);
   else if (auto CF = dyn_cast<ConstantFP>(&C))
     EntryBuilder->buildFConstant(Reg, *CF);
+  else if (isa<PoisonValue>(C))
+    EntryBuilder->buildPoison(Reg);
   else if (isa<UndefValue>(C))
     EntryBuilder->buildUndef(Reg);
   else if (isa<ConstantPointerNull>(C))