From 28edd62e82f84d8bd15464511940074b6066174d Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Mon, 6 Jan 2025 06:46:34 -0800 Subject: [PATCH 1/3] [TableGen][GlobalISel] Reorder atomic predicate to preserve the order Since there are no opcodes for atomic loads and stores comparing to SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the opcode predicate to make a logical combination of them. Otherwise when IPM_AtomicOrderingMMO is inserted after IPM_GenericPredicate, the patterns without predicates get a higher priority as IPM_AtomicOrderingMMO has higher priority than IPM_GenericPredicate. This is important to preserve order of aligned/unaligned patterns on X86 because aligned memory operations have an additional alignment predicate and should be checked first according to their placement in td file. Closes #121446 --- .../GlobalISelEmitter/GlobalISelEmitter.td | 6 +- .../TableGen/GlobalISelEmitter/HwModes.td | 8 +-- .../GlobalISelEmitter/MatchTableOptimizer.td | 4 +- .../GlobalISelEmitter/OverloadedPtr.td | 2 +- .../GlobalISelEmitter/atomic-store.td | 2 +- .../predicated-pattern-order.td | 55 +++++++++++++++++++ .../TableGen/GlobalISelEmitter/zero-reg.td | 2 +- llvm/test/TableGen/address-space-patfrags.td | 8 ++- llvm/utils/TableGen/GlobalISelEmitter.cpp | 21 ++++--- 9 files changed, 84 insertions(+), 24 deletions(-) create mode 100644 llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td diff --git a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td index 7c8181410d400..b7132bf2bcd8c 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter/GlobalISelEmitter.td @@ -950,8 +950,8 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), @@ -973,8 +973,8 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_p0s32, // NOOPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), @@ -996,8 +996,8 @@ def : Pat<(load GPR32:$src), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SEXTLOAD), -// NOOPT-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(2), // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// NOOPT-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(2), // NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), diff --git a/llvm/test/TableGen/GlobalISelEmitter/HwModes.td b/llvm/test/TableGen/GlobalISelEmitter/HwModes.td index 3588ba3979411..510368516739d 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/HwModes.td +++ b/llvm/test/TableGen/GlobalISelEmitter/HwModes.td @@ -131,8 +131,8 @@ class I Pat> // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode0), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s64, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID), @@ -149,8 +149,8 @@ class I Pat> // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode1), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID), @@ -173,8 +173,8 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1), // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode0), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_p0s64, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID), @@ -191,8 +191,8 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1), // CHECK-NEXT: GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode1), // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_p0s32, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID), diff --git a/llvm/test/TableGen/GlobalISelEmitter/MatchTableOptimizer.td b/llvm/test/TableGen/GlobalISelEmitter/MatchTableOptimizer.td index c4307258aae9a..6ac6703991c2d 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/MatchTableOptimizer.td +++ b/llvm/test/TableGen/GlobalISelEmitter/MatchTableOptimizer.td @@ -9,8 +9,8 @@ def LOAD8 : I<(outs GPR8:$dst), (ins GPR8:$src), []>; def LOAD32 : I<(outs GPR8:$dst), (ins GPR32:$src), []>; // CHECK: Label 1: @{{[0-9]+}} // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L1_ID:[0-9]+]]*/ GIMT_Encode4([[L1_AT:[0-9]+]]), +// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, -// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR8RegClassID), // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L2_ID:[0-9]+]]*/ GIMT_Encode4([[L2_AT:[0-9]+]]), // CHECK-NEXT: // MIs[0] src @@ -47,8 +47,8 @@ def LOAD16 : I<(outs GPR16:$dst), (ins GPR16:$src), []>; def LOAD16Imm : I<(outs GPR16:$dst), (ins GPR16:$src), []>; // CHECK: // Label 2: @{{[0-9]+}} // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L1_ID:[0-9]+]]*/ GIMT_Encode4([[L1_AT:[0-9]+]]), -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR16RegClassID), // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/16, // CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L2_ID:[0-9]+]]*/ GIMT_Encode4([[L2_AT:[0-9]+]]), diff --git a/llvm/test/TableGen/GlobalISelEmitter/OverloadedPtr.td b/llvm/test/TableGen/GlobalISelEmitter/OverloadedPtr.td index 31accba8b1847..43a121f94bd6c 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/OverloadedPtr.td +++ b/llvm/test/TableGen/GlobalISelEmitter/OverloadedPtr.td @@ -13,8 +13,8 @@ let TargetPrefix = "mytarget" in { // Check that iPTR in the destination DAG doesn't prevent the pattern from being imported. // CHECK: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // CHECK-NEXT: // MIs[0] src1 // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter/atomic-store.td b/llvm/test/TableGen/GlobalISelEmitter/atomic-store.td index 53b8670f47e63..99869cc4e8ef0 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/atomic-store.td +++ b/llvm/test/TableGen/GlobalISelEmitter/atomic-store.td @@ -6,8 +6,8 @@ include "GlobalISelEmitterCommon.td" def ST_ATOM_B32 : I<(outs), (ins GPR32Op:$val, GPR32Op:$ptr), []>; // GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), -// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(1), // GISEL-NEXT: GIM_CheckAtomicOrderingOrStrongerThan, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::Unordered, +// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(1), // GISEL-NEXT: // MIs[0] val // GISEL-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // GISEL-NEXT: // MIs[0] ptr diff --git a/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td b/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td new file mode 100644 index 0000000000000..19777d8a82173 --- /dev/null +++ b/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td @@ -0,0 +1,55 @@ +// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false -I %p/../../../include -I %p/../Common %s | FileCheck %s + +include "llvm/Target/Target.td" +include "GlobalISelEmitterCommon.td" + +// Check that IPM_GenericPredicate doesn't influence the final order of patterns. +// https://github.com/llvm/llvm-project/issues/121446 + +def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ + return true; +}]>{ + let GISelPredicateCode = [{ return true; }]; +} + +// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(45), // Rule ID 0 // +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, +// CHECK-NEXT: // MIs[0] src0 +// CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// CHECK-NEXT: // MIs[0] src1 +// CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, +// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_aligned_store), +// CHECK-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><><> => (MOVALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) +// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVALIGNED), +// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, +// CHECK-NEXT: // GIR_Coverage, 0, + +// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(86), // Rule ID 1 // +// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, +// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, +// CHECK-NEXT: // MIs[0] src0 +// CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, +// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// CHECK-NEXT: // MIs[0] src1 +// CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, +// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// CHECK-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><> => (MOVUNALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) +// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVUNALIGNED), +// CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, +// CHECK-NEXT: // GIR_Coverage, 1, + + +def MOVALIGNED : I<(outs), (ins GPR32:$src0, GPR32:$src1), + [(aligned_store GPR32:$src0, GPR32:$src1)]>; + + +def MOVUNALIGNED : I<(outs), (ins GPR32:$src0, GPR32:$src1), + [(store GPR32:$src0, GPR32:$src1)]>; + diff --git a/llvm/test/TableGen/GlobalISelEmitter/zero-reg.td b/llvm/test/TableGen/GlobalISelEmitter/zero-reg.td index 87e5432093377..dfbe7f902c011 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/zero-reg.td +++ b/llvm/test/TableGen/GlobalISelEmitter/zero-reg.td @@ -22,8 +22,8 @@ def INST : PredI<(outs GPR32:$dst), (ins GPR32:$src), []>; // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), -// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), diff --git a/llvm/test/TableGen/address-space-patfrags.td b/llvm/test/TableGen/address-space-patfrags.td index 582b97d55a518..a2611df048b06 100644 --- a/llvm/test/TableGen/address-space-patfrags.td +++ b/llvm/test/TableGen/address-space-patfrags.td @@ -60,9 +60,9 @@ def inst_d : Instruction { // GISEL: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 0 // // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), +// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // GISEL-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // GISEL-NEXT: GIM_CheckMemoryAddressSpace, /*MI*/0, /*MMO*/0, /*NumAddrSpace*/2, /*AddrSpace*/123, /*AddrSpace*//* 455(*/0xC7, 0x03/*)*/, -// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, def : Pat < (pat_frag_b GPR32:$src), (inst_b GPR32:$src) @@ -80,9 +80,9 @@ def : Pat < // GISEL: GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 1 // // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD), +// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // GISEL-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // GISEL-NEXT: GIM_CheckMemoryAlignment, /*MI*/0, /*MMO*/0, /*MinAlign*/2, -// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, def : Pat < (pat_frag_a GPR32:$src), (inst_a GPR32:$src) @@ -99,8 +99,8 @@ def truncstorei16_addrspace : PatFrag<(ops node:$val, node:$ptr), // GISEL: GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 2 // // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), -// GISEL-NEXT: GIM_CheckMemorySizeLessThanLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// GISEL-NEXT: GIM_CheckMemorySizeLessThanLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // GISEL-NEXT: // MIs[0] src0 // GISEL-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, def : Pat < @@ -112,6 +112,7 @@ def : Pat < // GISEL: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 3 // // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // GISEL-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, def : Pat < (store GPR32:$src0, GPR32:$src1), @@ -122,6 +123,7 @@ def : Pat < // GISEL: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 4 // // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // GISEL-NEXT: GIM_CheckMemorySizeLessThanLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(2), // GISEL-NEXT: GIM_CheckMemoryAddressSpace, /*MI*/0, /*MMO*/0, /*NumAddrSpace*/2, /*AddrSpace*/123, /*AddrSpace*//* 455(*/0xC7, 0x03/*)*/, diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 3b334ea4ce152..257b922bcfd59 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -765,6 +765,18 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( InsnMatcher.addPredicate(SrcGIOrNull); } + // Since there are no opcodes for atomic loads and stores comparing to + // SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the + // opcode predicate to make a logical combination of them. + if (SrcGIEquivOrNull && + SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) + InsnMatcher.addPredicate("NotAtomic"); + else if (SrcGIEquivOrNull && + SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { + InsnMatcher.addPredicate( + "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); + } + unsigned OpIdx = 0; for (const TypeSetByHwMode &VTy : Src.getExtTypes()) { // Results don't have a name unless they are the root node. The caller will @@ -827,15 +839,6 @@ Expected GlobalISelEmitter::createAndImportSelDAGMatcher( } } - if (SrcGIEquivOrNull && - SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) - InsnMatcher.addPredicate("NotAtomic"); - else if (SrcGIEquivOrNull && - SrcGIEquivOrNull->getValueAsBit("CheckMMOIsAtomic")) { - InsnMatcher.addPredicate( - "Unordered", AtomicOrderingMMOPredicateMatcher::AO_OrStronger); - } - if (Src.isLeaf()) { const Init *SrcInit = Src.getLeafValue(); if (const IntInit *SrcIntInit = dyn_cast(SrcInit)) { From 4a009fbdf43e5925122490414859b26b99cd0d73 Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Tue, 7 Jan 2025 06:33:04 -0800 Subject: [PATCH 2/3] Add test for the optimized generation --- .../predicated-pattern-order.td | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td b/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td index 19777d8a82173..ee667574d40da 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td +++ b/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td @@ -1,4 +1,5 @@ // RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false -I %p/../../../include -I %p/../Common %s | FileCheck %s +// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=true -I %p/../../../include -I %p/../Common %s | FileCheck -check-prefix=OPT %s include "llvm/Target/Target.td" include "GlobalISelEmitterCommon.td" @@ -45,6 +46,32 @@ def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ // CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, // CHECK-NEXT: // GIR_Coverage, 1, +// OPT: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(72), +// OPT-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), +// OPT-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, +// OPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, +// OPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, + +// OPT-NEXT: GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(47), // Rule ID 0 // +// OPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// OPT-NEXT: // MIs[0] src1 +// OPT-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, +// OPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// OPT-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_aligned_store), +// OPT-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><><> => (MOVALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) +// OPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVALIGNED), +// OPT-NEXT: GIR_RootConstrainSelectedInstOperands, +// OPT-NEXT: // GIR_Coverage, 0, + +// OPT: GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(71), // Rule ID 1 // +// OPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// OPT-NEXT: // MIs[0] src1 +// OPT-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, +// OPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), +// OPT-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><> => (MOVUNALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) +// OPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVUNALIGNED), +// OPT-NEXT: GIR_RootConstrainSelectedInstOperands, +// OPT-NEXT: // GIR_Coverage, 1, def MOVALIGNED : I<(outs), (ins GPR32:$src0, GPR32:$src1), [(aligned_store GPR32:$src0, GPR32:$src1)]>; From 3176ba8b804f270457c0f740c3a205ba085f2e4b Mon Sep 17 00:00:00 2001 From: Evgenii Kudriashov Date: Tue, 7 Jan 2025 06:37:32 -0800 Subject: [PATCH 3/3] Drop labels and IDs --- .../predicated-pattern-order.td | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td b/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td index ee667574d40da..ce420dbe01a27 100644 --- a/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td +++ b/llvm/test/TableGen/GlobalISelEmitter/predicated-pattern-order.td @@ -13,7 +13,7 @@ def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ let GISelPredicateCode = [{ return true; }]; } -// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(45), // Rule ID 0 // +// CHECK: GIM_Try // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, @@ -28,9 +28,9 @@ def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ // CHECK-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><><> => (MOVALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVALIGNED), // CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, -// CHECK-NEXT: // GIR_Coverage, 0, +// CHECK-NEXT: // GIR_Coverage -// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(86), // Rule ID 1 // +// CHECK: GIM_Try // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, @@ -44,15 +44,15 @@ def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ // CHECK-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><> => (MOVUNALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVUNALIGNED), // CHECK-NEXT: GIR_RootConstrainSelectedInstOperands, -// CHECK-NEXT: // GIR_Coverage, 1, +// CHECK-NEXT: // GIR_Coverage -// OPT: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(72), +// OPT: GIM_Try // OPT-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE), // OPT-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32, // OPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic, // OPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, -// OPT-NEXT: GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(47), // Rule ID 0 // +// OPT-NEXT: GIM_Try // OPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // OPT-NEXT: // MIs[0] src1 // OPT-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, @@ -61,9 +61,9 @@ def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ // OPT-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><><> => (MOVALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // OPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVALIGNED), // OPT-NEXT: GIR_RootConstrainSelectedInstOperands, -// OPT-NEXT: // GIR_Coverage, 0, +// OPT-NEXT: // GIR_Coverage -// OPT: GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(71), // Rule ID 1 // +// OPT: GIM_Try // OPT-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID), // OPT-NEXT: // MIs[0] src1 // OPT-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32, @@ -71,7 +71,7 @@ def aligned_store: PatFrag<(ops node:$v, node:$a), (store $v, $a), [{ // OPT-NEXT: // (st GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)<><> => (MOVUNALIGNED GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // OPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVUNALIGNED), // OPT-NEXT: GIR_RootConstrainSelectedInstOperands, -// OPT-NEXT: // GIR_Coverage, 1, +// OPT-NEXT: // GIR_Coverage def MOVALIGNED : I<(outs), (ins GPR32:$src0, GPR32:$src1), [(aligned_store GPR32:$src0, GPR32:$src1)]>;