From 6cd17502db5eb87c34672a0e0de88120e044ab30 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Tue, 22 Apr 2025 23:35:03 +0800 Subject: [PATCH 1/5] [RISCV] Add disjoint or patterns for vwadd[u].vv DAGCombiner::hoistLogicOpWithSameOpcodeHands will hoist (or disjoint (ext a), (ext b)) -> (ext (or disjoint a, b)) So this adds a pattern to match vwadd[u].vv in this case. We have to teach the combine to preserve the disjoint flag, and add a generic PatFrag for a disjoint or. This is meant to be a follow up to #136677 which would allow us to remove the target hook added there. --- .../include/llvm/Target/TargetSelectionDAG.td | 4 ++++ llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 +++- .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 19 +++++++++++++++++++ llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 14 ++++---------- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 9c241b6c4df0f..20ef517426cf8 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1113,6 +1113,10 @@ def not : PatFrag<(ops node:$in), (xor node:$in, -1)>; def vnot : PatFrag<(ops node:$in), (xor node:$in, immAllOnesV)>; def ineg : PatFrag<(ops node:$in), (sub 0, node:$in)>; +def or_disjoint : PatFrag<(ops node:$x, node:$y), (or node:$x, node:$y), [{ + return N->getFlags().hasDisjoint(); +}]>; + def zanyext : PatFrags<(ops node:$op), [(zext node:$op), (anyext node:$op)]>; diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index b175e35385ec6..8cfcd2be8c61c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -5982,7 +5982,9 @@ SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) { LegalTypes && !TLI.isTypeDesirableForOp(LogicOpcode, XVT)) return SDValue(); // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y) - SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y); + SDNodeFlags LogicFlags; + LogicFlags.setDisjoint(N->getFlags().hasDisjoint()); + SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y, LogicFlags); if (HandOpcode == ISD::SIGN_EXTEND_INREG) return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); return DAG.getNode(HandOpcode, DL, VT, Logic); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index b2c5261ae6c2d..71893e85bcb91 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -912,6 +912,25 @@ defm : VPatWidenBinarySDNode_VV_VX_WV_WX; defm : VPatWidenBinarySDNode_VV_VX_WV_WX; defm : VPatWidenBinarySDNode_VV_VX_WV_WX; +// DAGCombiner::hoistLogicOpWithSameOpcodeHands may hoist disjoint ors +// to (ext (or disjoint (a, b))) +multiclass VPatWidenOrDisjoint_VV { + foreach vtiToWti = AllWidenableIntVectors in { + defvar vti = vtiToWti.Vti; + defvar wti = vtiToWti.Wti; + let Predicates = !listconcat(GetVTypePredicates.Predicates, + GetVTypePredicates.Predicates) in { + def : Pat<(wti.Vector (extop (vti.Vector (or_disjoint vti.RegClass:$rs2, vti.RegClass:$rs1)))), + (!cast(instruction_name#"_VV_"#vti.LMul.MX) + (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, + vti.RegClass:$rs1, vti.AVL, vti.Log2SEW, TA_MA)>; + } + } +} +defm : VPatWidenOrDisjoint_VV; +defm : VPatWidenOrDisjoint_VV; +defm : VPatWidenOrDisjoint_VV; + defm : VPatWidenBinarySDNode_VV_VX_WV_WX; defm : VPatWidenBinarySDNode_VV_VX_WV_WX; defm : VPatWidenBinarySDNode_VV_VX_WV_WX; diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 3f5d42f89337b..149950484c477 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -1417,15 +1417,12 @@ define @vwaddu_vv_disjoint_or_add( %x.i8, %add } -; TODO: We could select vwaddu.vv, but when both arms of the or are the same -; DAGCombiner::hoistLogicOpWithSameOpcodeHands moves the zext above the or. define @vwaddu_vv_disjoint_or( %x.i16, %y.i16) { ; CHECK-LABEL: vwaddu_vv_disjoint_or: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma -; CHECK-NEXT: vor.vv v9, v8, v9 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vzext.vf2 v8, v9 +; CHECK-NEXT: vwaddu.vv v10, v8, v9 +; CHECK-NEXT: vmv1r.v v8, v10 ; CHECK-NEXT: ret %x.i32 = zext %x.i16 to %y.i32 = zext %y.i16 to @@ -1433,15 +1430,12 @@ define @vwaddu_vv_disjoint_or( %x.i16, %or } -; TODO: We could select vwadd.vv, but when both arms of the or are the same -; DAGCombiner::hoistLogicOpWithSameOpcodeHands moves the zext above the or. define @vwadd_vv_disjoint_or( %x.i16, %y.i16) { ; CHECK-LABEL: vwadd_vv_disjoint_or: ; CHECK: # %bb.0: ; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, ma -; CHECK-NEXT: vor.vv v9, v8, v9 -; CHECK-NEXT: vsetvli zero, zero, e32, m1, ta, ma -; CHECK-NEXT: vsext.vf2 v8, v9 +; CHECK-NEXT: vwadd.vv v10, v8, v9 +; CHECK-NEXT: vmv1r.v v8, v10 ; CHECK-NEXT: ret %x.i32 = sext %x.i16 to %y.i32 = sext %y.i16 to From 644fe31ee2b2dbe164fee51188faa36e3e21e3bc Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 23 Apr 2025 01:19:00 +0800 Subject: [PATCH 2/5] Don't propagate disjoint for inreg exts, use or_is_add, match SplatPat --- .../include/llvm/Target/TargetSelectionDAG.td | 4 --- llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +- .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 6 +++- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 31 +++++++++++++++++++ 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 20ef517426cf8..9c241b6c4df0f 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -1113,10 +1113,6 @@ def not : PatFrag<(ops node:$in), (xor node:$in, -1)>; def vnot : PatFrag<(ops node:$in), (xor node:$in, immAllOnesV)>; def ineg : PatFrag<(ops node:$in), (sub 0, node:$in)>; -def or_disjoint : PatFrag<(ops node:$x, node:$y), (or node:$x, node:$y), [{ - return N->getFlags().hasDisjoint(); -}]>; - def zanyext : PatFrags<(ops node:$op), [(zext node:$op), (anyext node:$op)]>; diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 8cfcd2be8c61c..f39951b1865ce 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -5983,7 +5983,8 @@ SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) { return SDValue(); // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y) SDNodeFlags LogicFlags; - LogicFlags.setDisjoint(N->getFlags().hasDisjoint()); + LogicFlags.setDisjoint(N->getFlags().hasDisjoint() && + ISD::isExtOpcode(HandOpcode)); SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y, LogicFlags); if (HandOpcode == ISD::SIGN_EXTEND_INREG) return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1)); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 71893e85bcb91..25c1d169502ea 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -920,10 +920,14 @@ multiclass VPatWidenOrDisjoint_VV { defvar wti = vtiToWti.Wti; let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in { - def : Pat<(wti.Vector (extop (vti.Vector (or_disjoint vti.RegClass:$rs2, vti.RegClass:$rs1)))), + def : Pat<(wti.Vector (extop (vti.Vector (or_is_add vti.RegClass:$rs2, vti.RegClass:$rs1)))), (!cast(instruction_name#"_VV_"#vti.LMul.MX) (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, vti.RegClass:$rs1, vti.AVL, vti.Log2SEW, TA_MA)>; + def : Pat<(wti.Vector (extop (vti.Vector (or_is_add vti.RegClass:$rs2, (SplatPat (XLenVT GPR:$rs1)))))), + (!cast(instruction_name#"_VX_"#vti.LMul.MX) + (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, + GPR:$rs1, vti.AVL, vti.Log2SEW, TA_MA)>; } } } diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index 149950484c477..ebccb18540bbd 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -1443,6 +1443,37 @@ define @vwadd_vv_disjoint_or( %x.i16, %or } +define @vwaddu_vx_disjoint_or( %x.i16, i16 %y.i16) { +; CHECK-LABEL: vwaddu_vx_disjoint_or: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwaddu.vx v9, v8, a0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %x.i32 = zext %x.i16 to + %y.head = insertelement poison, i16 %y.i16, i32 0 + %y.splat = shufflevector %y.head, poison, zeroinitializer + %y.i32 = zext %y.splat to + %or = or disjoint %x.i32, %y.i32 + ret %or +} + + +define @vwadd_vx_disjoint_or( %x.i16, i16 %y.i16) { +; CHECK-LABEL: vwadd_vx_disjoint_or: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e16, mf2, ta, ma +; CHECK-NEXT: vwadd.vx v9, v8, a0 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %x.i32 = sext %x.i16 to + %y.head = insertelement poison, i16 %y.i16, i32 0 + %y.splat = shufflevector %y.head, poison, zeroinitializer + %y.i32 = sext %y.splat to + %or = or disjoint %x.i32, %y.i32 + ret %or +} + define @vwaddu_wv_disjoint_or( %x.i32, %y.i16) { ; CHECK-LABEL: vwaddu_wv_disjoint_or: ; CHECK: # %bb.0: From f69c8e6c4f53b8ded3fd0c20eb35e25a289419bf Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 23 Apr 2025 14:14:14 +0800 Subject: [PATCH 3/5] Update multiclass name to include VX --- llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 25c1d169502ea..1e95ea1091c70 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -914,7 +914,7 @@ defm : VPatWidenBinarySDNode_VV_VX_WV_WX; // DAGCombiner::hoistLogicOpWithSameOpcodeHands may hoist disjoint ors // to (ext (or disjoint (a, b))) -multiclass VPatWidenOrDisjoint_VV { +multiclass VPatWidenOrDisjoint_VV_VX { foreach vtiToWti = AllWidenableIntVectors in { defvar vti = vtiToWti.Vti; defvar wti = vtiToWti.Wti; From 9599e60309532e715f17dafc8e9800aed59bfc0c Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 23 Apr 2025 14:14:57 +0800 Subject: [PATCH 4/5] Remove extraneous newline --- llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll index ebccb18540bbd..f94e46771f49c 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vwadd-sdnode.ll @@ -1458,7 +1458,6 @@ define @vwaddu_vx_disjoint_or( %x.i16, i16 ret %or } - define @vwadd_vx_disjoint_or( %x.i16, i16 %y.i16) { ; CHECK-LABEL: vwadd_vx_disjoint_or: ; CHECK: # %bb.0: From 288fb1f172fbe0f1549fbb5f4a9848e3b1895c50 Mon Sep 17 00:00:00 2001 From: Luke Lau Date: Wed, 23 Apr 2025 14:23:04 +0800 Subject: [PATCH 5/5] Fix records --- llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 1e95ea1091c70..aea125c5348dd 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -931,9 +931,9 @@ multiclass VPatWidenOrDisjoint_VV_VX { } } } -defm : VPatWidenOrDisjoint_VV; -defm : VPatWidenOrDisjoint_VV; -defm : VPatWidenOrDisjoint_VV; +defm : VPatWidenOrDisjoint_VV_VX; +defm : VPatWidenOrDisjoint_VV_VX; +defm : VPatWidenOrDisjoint_VV_VX; defm : VPatWidenBinarySDNode_VV_VX_WV_WX; defm : VPatWidenBinarySDNode_VV_VX_WV_WX;