-
Notifications
You must be signed in to change notification settings - Fork 14.9k
[AMDGPU] canCreateUndefOrPoisonForTargetNode - BFE_I32/U32 can't create poison/undef #154932
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Conversation
…te poison/undef Add AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode handler and tag BFE_I32/U32 nodes as they can only propagate poison, not create poison/undef. Fighting some of the remaining regressions in llvm#152107 - need advice on the v_round_f64 change
@llvm/pr-subscribers-backend-amdgpu Author: Simon Pilgrim (RKSimon) ChangesAdd AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode handler and tag BFE_I32/U32 nodes as they can only propagate poison, not create poison/undef. Fighting some of the remaining regressions in #152107 - need advice on the v_round_f64 change Full diff: https://github.com/llvm/llvm-project/pull/154932.diff 4 Files Affected:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 8ccd8fcc08d38..c048371b11d77 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -6135,6 +6135,19 @@ unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
}
}
+bool AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode(
+ SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
+ bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
+ unsigned Opcode = Op.getOpcode();
+ switch (Opcode) {
+ case AMDGPUISD::BFE_I32:
+ case AMDGPUISD::BFE_U32:
+ return false;
+ }
+ return TargetLowering::canCreateUndefOrPoisonForTargetNode(
+ Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
+}
+
bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
unsigned Depth) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index fd5d5b8dec431..78394ac9cd2dd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -323,6 +323,12 @@ class AMDGPUTargetLowering : public TargetLowering {
const MachineRegisterInfo &MRI,
unsigned Depth = 0) const override;
+ bool canCreateUndefOrPoisonForTargetNode(SDValue Op,
+ const APInt &DemandedElts,
+ const SelectionDAG &DAG,
+ bool PoisonOnly, bool ConsiderFlags,
+ unsigned Depth) const override;
+
bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
const SelectionDAG &DAG, bool SNaN = false,
unsigned Depth = 0) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 0df1a0fcc76ef..35913b9a21d30 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -1582,28 +1582,22 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; SI-NEXT: s_nop 1
; SI-NEXT: v_div_fmas_f64 v[4:5], v[12:13], v[6:7], v[10:11]
; SI-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s2, v5
-; SI-NEXT: s_bfe_u32 s0, s2, 0xb0014
-; SI-NEXT: s_add_i32 s3, s0, 0xfffffc01
-; SI-NEXT: s_mov_b32 s1, 0xfffff
-; SI-NEXT: s_mov_b32 s0, s6
-; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
-; SI-NEXT: v_not_b32_e32 v6, s0
-; SI-NEXT: v_and_b32_e32 v6, v4, v6
-; SI-NEXT: v_not_b32_e32 v7, s1
-; SI-NEXT: v_and_b32_e32 v5, v5, v7
-; SI-NEXT: s_and_b32 s0, s2, 0x80000000
-; SI-NEXT: s_cmp_lt_i32 s3, 0
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
-; SI-NEXT: v_mov_b32_e32 v7, s0
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT: s_cmp_gt_i32 s3, 51
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_mov_b32_e32 v7, s2
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s0, v4
+; SI-NEXT: v_readfirstlane_b32 s1, v5
+; SI-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; SI-NEXT: s_add_i32 s8, s2, 0xfffffc01
+; SI-NEXT: s_mov_b32 s3, 0xfffff
+; SI-NEXT: s_mov_b32 s2, s6
+; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s8
+; SI-NEXT: s_andn2_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT: s_and_b32 s9, s1, 0x80000000
+; SI-NEXT: s_cmp_lt_i32 s8, 0
+; SI-NEXT: s_cselect_b32 s2, 0, s2
+; SI-NEXT: s_cselect_b32 s3, s9, s3
+; SI-NEXT: s_cmp_gt_i32 s8, 51
+; SI-NEXT: s_cselect_b32 s1, s1, s3
+; SI-NEXT: s_cselect_b32 s0, s0, s2
+; SI-NEXT: v_fma_f64 v[0:1], -s[0:1], v[2:3], v[0:1]
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
@@ -1859,28 +1853,22 @@ define amdgpu_kernel void @fast_frem_f64(ptr addrspace(1) %out, ptr addrspace(1)
; SI-NEXT: v_mul_f64 v[6:7], v[0:1], v[4:5]
; SI-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
; SI-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: s_bfe_u32 s4, s6, 0xb0014
-; SI-NEXT: s_add_i32 s7, s4, 0xfffffc01
-; SI-NEXT: s_mov_b32 s5, 0xfffff
-; SI-NEXT: s_mov_b32 s4, s2
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], s7
-; SI-NEXT: v_not_b32_e32 v6, s4
-; SI-NEXT: v_and_b32_e32 v6, v4, v6
-; SI-NEXT: v_not_b32_e32 v7, s5
-; SI-NEXT: v_and_b32_e32 v5, v5, v7
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: s_cmp_lt_i32 s7, 0
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
-; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT: s_cmp_gt_i32 s7, 51
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_mov_b32_e32 v7, s6
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s5, v5
+; SI-NEXT: s_bfe_u32 s6, s5, 0xb0014
+; SI-NEXT: s_add_i32 s8, s6, 0xfffffc01
+; SI-NEXT: s_mov_b32 s7, 0xfffff
+; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s8
+; SI-NEXT: s_andn2_b64 s[6:7], s[4:5], s[6:7]
+; SI-NEXT: s_and_b32 s9, s5, 0x80000000
+; SI-NEXT: s_cmp_lt_i32 s8, 0
+; SI-NEXT: s_cselect_b32 s6, 0, s6
+; SI-NEXT: s_cselect_b32 s7, s9, s7
+; SI-NEXT: s_cmp_gt_i32 s8, 51
+; SI-NEXT: s_cselect_b32 s5, s5, s7
+; SI-NEXT: s_cselect_b32 s4, s4, s6
+; SI-NEXT: v_fma_f64 v[0:1], -s[4:5], v[2:3], v[0:1]
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -2109,28 +2097,22 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace(
; SI-NEXT: v_mul_f64 v[6:7], v[0:1], v[4:5]
; SI-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1]
; SI-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7]
-; SI-NEXT: v_readfirstlane_b32 s6, v5
-; SI-NEXT: s_bfe_u32 s4, s6, 0xb0014
-; SI-NEXT: s_add_i32 s7, s4, 0xfffffc01
-; SI-NEXT: s_mov_b32 s5, 0xfffff
-; SI-NEXT: s_mov_b32 s4, s2
-; SI-NEXT: s_lshr_b64 s[4:5], s[4:5], s7
-; SI-NEXT: v_not_b32_e32 v6, s4
-; SI-NEXT: v_and_b32_e32 v6, v4, v6
-; SI-NEXT: v_not_b32_e32 v7, s5
-; SI-NEXT: v_and_b32_e32 v5, v5, v7
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: s_cmp_lt_i32 s7, 0
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
-; SI-NEXT: v_mov_b32_e32 v7, s4
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT: s_cmp_gt_i32 s7, 51
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_mov_b32_e32 v7, s6
-; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc
-; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s4, v4
+; SI-NEXT: v_readfirstlane_b32 s5, v5
+; SI-NEXT: s_bfe_u32 s6, s5, 0xb0014
+; SI-NEXT: s_add_i32 s8, s6, 0xfffffc01
+; SI-NEXT: s_mov_b32 s7, 0xfffff
+; SI-NEXT: s_mov_b32 s6, s2
+; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s8
+; SI-NEXT: s_andn2_b64 s[6:7], s[4:5], s[6:7]
+; SI-NEXT: s_and_b32 s9, s5, 0x80000000
+; SI-NEXT: s_cmp_lt_i32 s8, 0
+; SI-NEXT: s_cselect_b32 s6, 0, s6
+; SI-NEXT: s_cselect_b32 s7, s9, s7
+; SI-NEXT: s_cmp_gt_i32 s8, 51
+; SI-NEXT: s_cselect_b32 s5, s5, s7
+; SI-NEXT: s_cselect_b32 s4, s4, s6
+; SI-NEXT: v_fma_f64 v[0:1], -s[4:5], v[2:3], v[0:1]
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
@@ -5251,27 +5233,22 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: s_nop 1
; SI-NEXT: v_div_fmas_f64 v[8:9], v[16:17], v[10:11], v[14:15]
; SI-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3]
-; SI-NEXT: v_readfirstlane_b32 s8, v9
-; SI-NEXT: s_bfe_u32 s0, s8, 0xb0014
-; SI-NEXT: s_add_i32 s9, s0, 0xfffffc01
+; SI-NEXT: v_readfirstlane_b32 s0, v8
+; SI-NEXT: v_readfirstlane_b32 s1, v9
+; SI-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; SI-NEXT: s_add_i32 s10, s2, 0xfffffc01
; SI-NEXT: s_mov_b32 s3, 0xfffff
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s9
-; SI-NEXT: v_not_b32_e32 v10, s0
-; SI-NEXT: v_and_b32_e32 v10, v8, v10
-; SI-NEXT: v_not_b32_e32 v11, s1
-; SI-NEXT: v_and_b32_e32 v9, v9, v11
-; SI-NEXT: s_and_b32 s0, s8, 0x80000000
-; SI-NEXT: s_cmp_lt_i32 s9, 0
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
-; SI-NEXT: v_mov_b32_e32 v11, s0
-; SI-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; SI-NEXT: s_cmp_gt_i32 s9, 51
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_mov_b32_e32 v11, s8
-; SI-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc
-; SI-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; SI-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3]
+; SI-NEXT: s_mov_b32 s2, s6
+; SI-NEXT: s_lshr_b64 s[8:9], s[2:3], s10
+; SI-NEXT: s_andn2_b64 s[8:9], s[0:1], s[8:9]
+; SI-NEXT: s_and_b32 s11, s1, 0x80000000
+; SI-NEXT: s_cmp_lt_i32 s10, 0
+; SI-NEXT: s_cselect_b32 s8, 0, s8
+; SI-NEXT: s_cselect_b32 s9, s11, s9
+; SI-NEXT: s_cmp_gt_i32 s10, 51
+; SI-NEXT: s_cselect_b32 s1, s1, s9
+; SI-NEXT: s_cselect_b32 s0, s0, s8
+; SI-NEXT: v_fma_f64 v[2:3], -s[0:1], v[6:7], v[2:3]
; SI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1]
; SI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7]
; SI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0
@@ -5287,26 +5264,20 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: s_nop 1
; SI-NEXT: v_div_fmas_f64 v[6:7], v[14:15], v[8:9], v[12:13]
; SI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1]
-; SI-NEXT: v_readfirstlane_b32 s8, v7
-; SI-NEXT: s_bfe_u32 s0, s8, 0xb0014
-; SI-NEXT: s_add_i32 s9, s0, 0xfffffc01
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s9
-; SI-NEXT: v_not_b32_e32 v8, s0
-; SI-NEXT: v_and_b32_e32 v8, v6, v8
-; SI-NEXT: v_not_b32_e32 v9, s1
-; SI-NEXT: v_and_b32_e32 v7, v7, v9
-; SI-NEXT: s_and_b32 s0, s8, 0x80000000
-; SI-NEXT: s_cmp_lt_i32 s9, 0
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc
-; SI-NEXT: v_mov_b32_e32 v9, s0
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
-; SI-NEXT: s_cmp_gt_i32 s9, 51
-; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: v_mov_b32_e32 v9, s8
-; SI-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc
-; SI-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
-; SI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1]
+; SI-NEXT: v_readfirstlane_b32 s0, v6
+; SI-NEXT: v_readfirstlane_b32 s1, v7
+; SI-NEXT: s_bfe_u32 s8, s1, 0xb0014
+; SI-NEXT: s_addk_i32 s8, 0xfc01
+; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s8
+; SI-NEXT: s_andn2_b64 s[2:3], s[0:1], s[2:3]
+; SI-NEXT: s_and_b32 s9, s1, 0x80000000
+; SI-NEXT: s_cmp_lt_i32 s8, 0
+; SI-NEXT: s_cselect_b32 s2, 0, s2
+; SI-NEXT: s_cselect_b32 s3, s9, s3
+; SI-NEXT: s_cmp_gt_i32 s8, 51
+; SI-NEXT: s_cselect_b32 s1, s1, s3
+; SI-NEXT: s_cselect_b32 s0, s0, s2
+; SI-NEXT: v_fma_f64 v[0:1], -s[0:1], v[4:5], v[0:1]
; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
index af914bd4043cf..355f77acfd302 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll
@@ -76,12 +76,13 @@ define amdgpu_kernel void @v_round_f64(ptr addrspace(1) %out, ptr addrspace(1) %
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_movk_i32 s4, 0xfc01
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_mov_b32 s3, 0xfffff
; SI-NEXT: v_mov_b32_e32 v8, 0x3ff00000
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_bfe_u32 v4, v3, 20, 11
-; SI-NEXT: v_add_i32_e32 v6, vcc, 0xfffffc01, v4
+; SI-NEXT: v_add_i32_e32 v6, vcc, s4, v4
; SI-NEXT: v_lshr_b64 v[4:5], s[2:3], v6
; SI-NEXT: v_and_b32_e32 v7, 0x80000000, v3
; SI-NEXT: v_not_b32_e32 v5, v5
|
; SI-NEXT: s_mov_b32 s2, -1 | ||
; SI-NEXT: s_mov_b32 s3, 0xfffff | ||
; SI-NEXT: v_mov_b32_e32 v8, 0x3ff00000 | ||
; SI-NEXT: s_waitcnt vmcnt(0) | ||
; SI-NEXT: v_bfe_u32 v4, v3, 20, 11 | ||
; SI-NEXT: v_add_i32_e32 v6, vcc, 0xfffffc01, v4 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't know how anything you do in the DAG would cause this regression
Add AMDGPUTargetLowering::canCreateUndefOrPoisonForTargetNode handler and tag BFE_I32/U32 nodes as they can only propagate poison, not create poison/undef.
Fighting some of the remaining regressions in #152107 - need advice on the v_round_f64 change