[LICM] Improve LICM when calls only change Inaccessible memory

Extend `MemorySSA`’s clobber query to better distinguish calls that
access inaccessible memory improving code motion opportunities in loops.

If both calls dont clobber Inaccessible Memory Location it can return
from instructionClobbersQuery without setting ModeRefInfo.
Otherwise it relies in the default behaviour to set ModRefInfo to
Read and Write.

This enables LICM to hoist calls that modify inaccessible memory,
improving code motion opportunities in loops.

Author: CarolineConcatto

llvm/lib/Analysis/MemorySSA.cpp

@@ -277,6 +277,17 @@ static bool areLoadsReorderable(const LoadInst *Use,
   return !(SeqCstUse || MayClobberIsAcquire);
 }
 
+bool hasInaccessibleMemoryClobber(const CallBase *CallFirst,
+                                  const CallBase *CallSecond) {
+
+  MemoryEffects ME1 = CallFirst->getMemoryEffects();
+  MemoryEffects ME2 = CallSecond->getMemoryEffects();
+  if (CallFirst->onlyAccessesInaccessibleMemory() ||
+      CallSecond->onlyAccessesInaccessibleMemory())
+    return !(ME1 & ME2 & MemoryEffects::writeOnly()).onlyReadsMemory();
+  return true;
+}
+
 template <typename AliasAnalysisType>
 static bool
 instructionClobbersQuery(const MemoryDef *MD, const MemoryLocation &UseLoc,
@@ -311,6 +322,9 @@ instructionClobbersQuery(const MemoryDef *MD, const MemoryLocation &UseLoc,
   }
 
   if (auto *CB = dyn_cast_or_null<CallBase>(UseInst)) {
+    if (auto *CU = dyn_cast_or_null<CallBase>(DefInst))
+      if (!hasInaccessibleMemoryClobber(CB, CU))
+        return false;
     ModRefInfo I = AA.getModRefInfo(DefInst, CB);
     return isModOrRefSet(I);
   }

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll

@@ -885,7 +885,7 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; SI-NEXT:    s_mov_b64 s[0:1], exec
 ; SI-NEXT:    s_wqm_b64 exec, exec
 ; SI-NEXT:    v_cvt_i32_f32_e32 v0, v0
-; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; SI-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; SI-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
@@ -894,24 +894,24 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 .LBB7_9
 ; SI-NEXT:  ; %bb.2: ; %.demote0
-; SI-NEXT:    s_wqm_b64 s[6:7], s[0:1]
-; SI-NEXT:    s_and_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_wqm_b64 s[4:5], s[0:1]
+; SI-NEXT:    s_and_b64 exec, exec, s[4:5]
 ; SI-NEXT:  .LBB7_3: ; %.continue0.preheader
 ; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
+; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; SI-NEXT:    s_mov_b64 s[2:3], 0
-; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    v_mov_b32_e32 v0, s6
 ; SI-NEXT:    s_branch .LBB7_5
 ; SI-NEXT:  .LBB7_4: ; %.continue1
 ; SI-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; SI-NEXT:    s_or_b64 exec, exec, s[4:5]
+; SI-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; SI-NEXT:    v_add_u32_e32 v0, vcc, 1, v0
 ; SI-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v1
 ; SI-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; SI-NEXT:    s_cbranch_execz .LBB7_8
 ; SI-NEXT:  .LBB7_5: ; %.continue0
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
-; SI-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s[4:5]
 ; SI-NEXT:    v_mov_b32_e32 v3, v2
 ; SI-NEXT:    s_nop 1
@@ -920,19 +920,19 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; SI-NEXT:    v_subrev_f32_dpp v2, v2, v3 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; SI-NEXT:    ; kill: def $vgpr2 killed $vgpr2 killed $exec
 ; SI-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; SI-NEXT:    s_and_b64 s[4:5], s[0:1], vcc
-; SI-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; SI-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
-; SI-NEXT:    s_xor_b64 s[4:5], exec, s[6:7]
+; SI-NEXT:    s_and_b64 s[6:7], s[4:5], vcc
+; SI-NEXT:    s_xor_b64 s[6:7], s[6:7], -1
+; SI-NEXT:    s_and_saveexec_b64 s[8:9], s[6:7]
+; SI-NEXT:    s_xor_b64 s[6:7], exec, s[8:9]
 ; SI-NEXT:    s_cbranch_execz .LBB7_4
 ; SI-NEXT:  ; %bb.6: ; %.demote1
 ; SI-NEXT:    ; in Loop: Header=BB7_5 Depth=1
 ; SI-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; SI-NEXT:    s_cbranch_scc0 .LBB7_9
 ; SI-NEXT:  ; %bb.7: ; %.demote1
 ; SI-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; SI-NEXT:    s_wqm_b64 s[6:7], s[0:1]
-; SI-NEXT:    s_and_b64 exec, exec, s[6:7]
+; SI-NEXT:    s_wqm_b64 s[8:9], s[0:1]
+; SI-NEXT:    s_and_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_branch .LBB7_4
 ; SI-NEXT:  .LBB7_8: ; %.return
 ; SI-NEXT:    s_or_b64 exec, exec, s[2:3]
@@ -951,7 +951,7 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX9-NEXT:    s_mov_b64 s[0:1], exec
 ; GFX9-NEXT:    s_wqm_b64 exec, exec
 ; GFX9-NEXT:    v_cvt_i32_f32_e32 v0, v0
-; GFX9-NEXT:    s_mov_b32 s4, 0
+; GFX9-NEXT:    s_mov_b32 s6, 0
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
 ; GFX9-NEXT:    s_and_saveexec_b64 s[2:3], vcc
 ; GFX9-NEXT:    s_xor_b64 s[2:3], exec, s[2:3]
@@ -960,24 +960,24 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB7_9
 ; GFX9-NEXT:  ; %bb.2: ; %.demote0
-; GFX9-NEXT:    s_wqm_b64 s[6:7], s[0:1]
-; GFX9-NEXT:    s_and_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_wqm_b64 s[4:5], s[0:1]
+; GFX9-NEXT:    s_and_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:  .LBB7_3: ; %.continue0.preheader
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    s_mov_b64 s[2:3], 0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-NEXT:    v_mov_b32_e32 v0, s6
 ; GFX9-NEXT:    s_branch .LBB7_5
 ; GFX9-NEXT:  .LBB7_4: ; %.continue1
 ; GFX9-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX9-NEXT:    v_add_u32_e32 v0, 1, v0
 ; GFX9-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v1
 ; GFX9-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[2:3]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_8
 ; GFX9-NEXT:  .LBB7_5: ; %.continue0
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_mov_b64 s[4:5], s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX9-NEXT:    s_nop 1
@@ -986,19 +986,19 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX9-NEXT:    v_subrev_f32_dpp v2, v2, v3 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX9-NEXT:    ; kill: def $vgpr2 killed $vgpr2 killed $exec
 ; GFX9-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX9-NEXT:    s_and_b64 s[4:5], s[0:1], vcc
-; GFX9-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GFX9-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
-; GFX9-NEXT:    s_xor_b64 s[4:5], exec, s[6:7]
+; GFX9-NEXT:    s_and_b64 s[6:7], s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[6:7], s[6:7], -1
+; GFX9-NEXT:    s_and_saveexec_b64 s[8:9], s[6:7]
+; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[8:9]
 ; GFX9-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX9-NEXT:  ; %bb.6: ; %.demote1
 ; GFX9-NEXT:    ; in Loop: Header=BB7_5 Depth=1
 ; GFX9-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB7_9
 ; GFX9-NEXT:  ; %bb.7: ; %.demote1
 ; GFX9-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; GFX9-NEXT:    s_wqm_b64 s[6:7], s[0:1]
-; GFX9-NEXT:    s_and_b64 exec, exec, s[6:7]
+; GFX9-NEXT:    s_wqm_b64 s[8:9], s[0:1]
+; GFX9-NEXT:    s_and_b64 exec, exec, s[8:9]
 ; GFX9-NEXT:    s_branch .LBB7_4
 ; GFX9-NEXT:  .LBB7_8: ; %.return
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[2:3]
@@ -1031,37 +1031,37 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX10-32-NEXT:  .LBB7_3: ; %.continue0.preheader
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s2
 ; GFX10-32-NEXT:    v_mov_b32_e32 v0, s1
+; GFX10-32-NEXT:    s_mov_b32 s2, s0
 ; GFX10-32-NEXT:    s_branch .LBB7_5
 ; GFX10-32-NEXT:  .LBB7_4: ; %.continue1
 ; GFX10-32-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s3
 ; GFX10-32-NEXT:    v_add_nc_u32_e32 v0, 1, v0
 ; GFX10-32-NEXT:    v_cmp_ge_i32_e32 vcc_lo, v0, v1
 ; GFX10-32-NEXT:    s_or_b32 s1, vcc_lo, s1
 ; GFX10-32-NEXT:    s_andn2_b32 exec_lo, exec_lo, s1
 ; GFX10-32-NEXT:    s_cbranch_execz .LBB7_8
 ; GFX10-32-NEXT:  .LBB7_5: ; %.continue0
 ; GFX10-32-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-32-NEXT:    s_mov_b32 s2, s0
 ; GFX10-32-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s2
 ; GFX10-32-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX10-32-NEXT:    v_mov_b32_dpp v3, v3 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX10-32-NEXT:    v_subrev_f32_dpp v2, v2, v3 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX10-32-NEXT:    ; kill: def $vgpr2 killed $vgpr2 killed $exec
 ; GFX10-32-NEXT:    v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX10-32-NEXT:    s_and_b32 s2, s0, vcc_lo
-; GFX10-32-NEXT:    s_xor_b32 s2, s2, -1
-; GFX10-32-NEXT:    s_and_saveexec_b32 s3, s2
-; GFX10-32-NEXT:    s_xor_b32 s2, exec_lo, s3
+; GFX10-32-NEXT:    s_and_b32 s3, s2, vcc_lo
+; GFX10-32-NEXT:    s_xor_b32 s3, s3, -1
+; GFX10-32-NEXT:    s_and_saveexec_b32 s4, s3
+; GFX10-32-NEXT:    s_xor_b32 s3, exec_lo, s4
 ; GFX10-32-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX10-32-NEXT:  ; %bb.6: ; %.demote1
 ; GFX10-32-NEXT:    ; in Loop: Header=BB7_5 Depth=1
 ; GFX10-32-NEXT:    s_andn2_b32 s0, s0, exec_lo
 ; GFX10-32-NEXT:    s_cbranch_scc0 .LBB7_9
 ; GFX10-32-NEXT:  ; %bb.7: ; %.demote1
 ; GFX10-32-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; GFX10-32-NEXT:    s_wqm_b32 s3, s0
-; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s3
+; GFX10-32-NEXT:    s_wqm_b32 s4, s0
+; GFX10-32-NEXT:    s_and_b32 exec_lo, exec_lo, s4
 ; GFX10-32-NEXT:    s_branch .LBB7_4
 ; GFX10-32-NEXT:  .LBB7_8: ; %.return
 ; GFX10-32-NEXT:    s_or_b32 exec_lo, exec_lo, s1
@@ -1094,41 +1094,41 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
 ; GFX10-64-NEXT:  .LBB7_3: ; %.continue0.preheader
 ; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
 ; GFX10-64-NEXT:    v_mov_b32_e32 v0, s4
-; GFX10-64-NEXT:    s_mov_b64 s[2:3], 0
+; GFX10-64-NEXT:    s_mov_b64 s[2:3], s[0:1]
+; GFX10-64-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX10-64-NEXT:    s_branch .LBB7_5
 ; GFX10-64-NEXT:  .LBB7_4: ; %.continue1
 ; GFX10-64-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; GFX10-64-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX10-64-NEXT:    s_or_b64 exec, exec, s[6:7]
 ; GFX10-64-NEXT:    v_add_nc_u32_e32 v0, 1, v0
 ; GFX10-64-NEXT:    v_cmp_ge_i32_e32 vcc, v0, v1
-; GFX10-64-NEXT:    s_or_b64 s[2:3], vcc, s[2:3]
-; GFX10-64-NEXT:    s_andn2_b64 exec, exec, s[2:3]
+; GFX10-64-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
+; GFX10-64-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GFX10-64-NEXT:    s_cbranch_execz .LBB7_8
 ; GFX10-64-NEXT:  .LBB7_5: ; %.continue0
 ; GFX10-64-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-64-NEXT:    s_mov_b64 s[4:5], s[0:1]
-; GFX10-64-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s[4:5]
+; GFX10-64-NEXT:    v_cndmask_b32_e64 v2, v0, 0, s[2:3]
 ; GFX10-64-NEXT:    v_mov_b32_e32 v3, v2
 ; GFX10-64-NEXT:    v_mov_b32_dpp v3, v3 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX10-64-NEXT:    v_subrev_f32_dpp v2, v2, v3 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GFX10-64-NEXT:    ; kill: def $vgpr2 killed $vgpr2 killed $exec
 ; GFX10-64-NEXT:    v_cmp_eq_f32_e32 vcc, 0, v2
-; GFX10-64-NEXT:    s_and_b64 s[4:5], s[0:1], vcc
-; GFX10-64-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GFX10-64-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
-; GFX10-64-NEXT:    s_xor_b64 s[4:5], exec, s[6:7]
+; GFX10-64-NEXT:    s_and_b64 s[6:7], s[2:3], vcc
+; GFX10-64-NEXT:    s_xor_b64 s[6:7], s[6:7], -1
+; GFX10-64-NEXT:    s_and_saveexec_b64 s[8:9], s[6:7]
+; GFX10-64-NEXT:    s_xor_b64 s[6:7], exec, s[8:9]
 ; GFX10-64-NEXT:    s_cbranch_execz .LBB7_4
 ; GFX10-64-NEXT:  ; %bb.6: ; %.demote1
 ; GFX10-64-NEXT:    ; in Loop: Header=BB7_5 Depth=1
 ; GFX10-64-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
 ; GFX10-64-NEXT:    s_cbranch_scc0 .LBB7_9
 ; GFX10-64-NEXT:  ; %bb.7: ; %.demote1
 ; GFX10-64-NEXT:    ; in Loop: Header=BB7_5 Depth=1
-; GFX10-64-NEXT:    s_wqm_b64 s[6:7], s[0:1]
-; GFX10-64-NEXT:    s_and_b64 exec, exec, s[6:7]
+; GFX10-64-NEXT:    s_wqm_b64 s[8:9], s[0:1]
+; GFX10-64-NEXT:    s_and_b64 exec, exec, s[8:9]
 ; GFX10-64-NEXT:    s_branch .LBB7_4
 ; GFX10-64-NEXT:  .LBB7_8: ; %.return
-; GFX10-64-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX10-64-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX10-64-NEXT:    s_and_b64 exec, exec, s[0:1]
 ; GFX10-64-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; GFX10-64-NEXT:    v_bfrev_b32_e32 v1, 60