From d298e8ce49948e7c7b84016f1babbdbd8f02e720 Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Wed, 19 Mar 2025 16:50:37 -0700 Subject: [PATCH 1/3] [RISCV] Fix incorrect slide offset when using vnsrl to de-interleave Fix #132071 --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 21 +++++++++++++++---- .../rvv/fixed-vectors-shuffle-deinterleave.ll | 6 ++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 48d8fc23dc1bb..4a0de99428605 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -5545,12 +5545,25 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, })) { // Narrow each source and concatenate them. // FIXME: For small LMUL it is better to concatenate first. - MVT HalfVT = VT.getHalfNumVectorElementsVT(); + MVT EltVT = VT.getVectorElementType(); + auto EltCnt = VT.getVectorElementCount(); + MVT SubVT = + MVT::getVectorVT(EltVT, EltCnt.divideCoefficientBy(Factor)); + SDValue Lo = - getDeinterleaveShiftAndTrunc(DL, HalfVT, V1, Factor, Index, DAG); + getDeinterleaveShiftAndTrunc(DL, SubVT, V1, Factor, Index, DAG); SDValue Hi = - getDeinterleaveShiftAndTrunc(DL, HalfVT, V2, Factor, Index, DAG); - return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); + getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG); + + MVT NewVT = SubVT.getDoubleNumVectorElementsVT(); + SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, Lo, Hi); + for (unsigned F = Factor; F > 2; F >>= 1) { + SDValue Undef = DAG.getUNDEF(NewVT); + NewVT = NewVT.getDoubleNumVectorElementsVT(); + Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, Concat, Undef); + } + + return Concat; } } } diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll index 5e6d7c1eedb76..0b4231cedcab5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll @@ -378,8 +378,9 @@ define void @deinterleave4_0_i8_two_source(ptr %in0, ptr %in1, ptr %out) { ; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma ; CHECK-NEXT: vnsrl.wi v8, v8, 0 ; CHECK-NEXT: vnsrl.wi v9, v9, 0 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 2 ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vslideup.vi v9, v8, 4 ; CHECK-NEXT: vse8.v v9, (a2) ; CHECK-NEXT: ret entry: @@ -402,8 +403,9 @@ define void @deinterleave4_8_i8_two_source(ptr %in0, ptr %in1, ptr %out) { ; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma ; CHECK-NEXT: vnsrl.wi v8, v8, 0 ; CHECK-NEXT: vnsrl.wi v9, v9, 0 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vslideup.vi v9, v8, 2 ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vslideup.vi v9, v8, 4 ; CHECK-NEXT: vse8.v v9, (a2) ; CHECK-NEXT: ret entry: From 1400fa946449e0d78b72f6c7a8b399f4a5dad659 Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Wed, 19 Mar 2025 21:17:33 -0700 Subject: [PATCH 2/3] fixup! Use INSERT_SUBVECTOR intead of using many CONCAT_VECTORS --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 15 ++++++--------- .../rvv/fixed-vectors-shuffle-deinterleave.ll | 4 ++-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 4a0de99428605..1b9b16fd0dc9f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -5555,15 +5555,12 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, SDValue Hi = getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG); - MVT NewVT = SubVT.getDoubleNumVectorElementsVT(); - SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, Lo, Hi); - for (unsigned F = Factor; F > 2; F >>= 1) { - SDValue Undef = DAG.getUNDEF(NewVT); - NewVT = NewVT.getDoubleNumVectorElementsVT(); - Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, Concat, Undef); - } - - return Concat; + SDValue Vec = DAG.getUNDEF(VT); + Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, Lo, + DAG.getVectorIdxConstant(0, DL)); + return DAG.getNode( + ISD::INSERT_SUBVECTOR, DL, VT, Vec, Hi, + DAG.getVectorIdxConstant(SubVT.getVectorMinNumElements(), DL)); } } } diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll index 0b4231cedcab5..6e77d5cad0710 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll @@ -378,7 +378,7 @@ define void @deinterleave4_0_i8_two_source(ptr %in0, ptr %in1, ptr %out) { ; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma ; CHECK-NEXT: vnsrl.wi v8, v8, 0 ; CHECK-NEXT: vnsrl.wi v9, v9, 0 -; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma ; CHECK-NEXT: vslideup.vi v9, v8, 2 ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; CHECK-NEXT: vse8.v v9, (a2) @@ -403,7 +403,7 @@ define void @deinterleave4_8_i8_two_source(ptr %in0, ptr %in1, ptr %out) { ; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma ; CHECK-NEXT: vnsrl.wi v8, v8, 0 ; CHECK-NEXT: vnsrl.wi v9, v9, 0 -; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma +; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma ; CHECK-NEXT: vslideup.vi v9, v8, 2 ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; CHECK-NEXT: vse8.v v9, (a2) From d1146cf3fd0670717d475bdf83f2a424843e6550 Mon Sep 17 00:00:00 2001 From: Min Hsu Date: Wed, 19 Mar 2025 21:37:52 -0700 Subject: [PATCH 3/3] fixup! Address review comments --- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 13 ++++++++----- .../RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 1b9b16fd0dc9f..132faf5b85c1a 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -5555,12 +5555,15 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, SDValue Hi = getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG); + SDValue Concat = + DAG.getNode(ISD::CONCAT_VECTORS, DL, + SubVT.getDoubleNumVectorElementsVT(), Lo, Hi); + if (Factor == 2) + return Concat; + SDValue Vec = DAG.getUNDEF(VT); - Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, Lo, - DAG.getVectorIdxConstant(0, DL)); - return DAG.getNode( - ISD::INSERT_SUBVECTOR, DL, VT, Vec, Hi, - DAG.getVectorIdxConstant(SubVT.getVectorMinNumElements(), DL)); + return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, Concat, + DAG.getVectorIdxConstant(0, DL)); } } } diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll index 6e77d5cad0710..0b4231cedcab5 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-shuffle-deinterleave.ll @@ -378,7 +378,7 @@ define void @deinterleave4_0_i8_two_source(ptr %in0, ptr %in1, ptr %out) { ; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma ; CHECK-NEXT: vnsrl.wi v8, v8, 0 ; CHECK-NEXT: vnsrl.wi v9, v9, 0 -; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; CHECK-NEXT: vslideup.vi v9, v8, 2 ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; CHECK-NEXT: vse8.v v9, (a2) @@ -403,7 +403,7 @@ define void @deinterleave4_8_i8_two_source(ptr %in0, ptr %in1, ptr %out) { ; CHECK-NEXT: vsetvli zero, zero, e8, mf8, ta, ma ; CHECK-NEXT: vnsrl.wi v8, v8, 0 ; CHECK-NEXT: vnsrl.wi v9, v9, 0 -; CHECK-NEXT: vsetivli zero, 4, e8, mf2, tu, ma +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, ma ; CHECK-NEXT: vslideup.vi v9, v8, 2 ; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma ; CHECK-NEXT: vse8.v v9, (a2)