Reapply [AMDGPU] prevent shrinking udiv/urem if either operand exceeds signed max (llvm#119325) (#73)

This reverts commit 254d206.

+Added a fix in ExpandDivRem24 to disqualify if DivNumBits exceed 24.

Original commit & msg:
ce6e955.
Handle signed and unsigned path differently in getDivNumBits. Using
computeKnownBits, this rejects shrinking unsigned div/rem if operands
exceed signed max since we know NumSignBits will be always 0.

(cherry picked from commit 463e93b)

Co-authored-by: choikwa <[email protected]>

Author: bcahoon

llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp

@@ -1194,18 +1194,37 @@ int AMDGPUCodeGenPrepareImpl::getDivNumBits(BinaryOperator &I, Value *Num,
                                             Value *Den, unsigned AtLeast,
                                             bool IsSigned) const {
   const DataLayout &DL = Mod->getDataLayout();
-  unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
-  if (LHSSignBits < AtLeast)
-    return -1;
+  assert(Num->getType()->getScalarSizeInBits() ==
+         Den->getType()->getScalarSizeInBits());
+  unsigned SSBits = Num->getType()->getScalarSizeInBits();
+  if (IsSigned) {
+    unsigned RHSSignBits = ComputeNumSignBits(Den, DL, 0, AC, &I);
+    if (RHSSignBits < AtLeast)
+      return -1;
+
+    unsigned LHSSignBits = ComputeNumSignBits(Num, DL, 0, AC, &I);
+    if (LHSSignBits < AtLeast)
+      return -1;
+
+    unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
+    unsigned DivBits = SSBits - SignBits + 1;
+    return DivBits; // a SignBit needs to be reserved for shrinking
+  }
+
+  // All bits are used for unsigned division for Num or Den in range
+  // (SignedMax, UnsignedMax].
+  KnownBits Known = computeKnownBits(Den, DL, 0, AC, &I);
+  if (Known.isNegative() || !Known.isNonNegative())
+    return SSBits;
+  unsigned RHSSignBits = Known.countMinLeadingZeros();
 
-  unsigned RHSSignBits = ComputeNumSignBits(Den, DL, 0, AC, &I);
-  if (RHSSignBits < AtLeast)
-    return -1;
+  Known = computeKnownBits(Num, DL, 0, AC, &I);
+  if (Known.isNegative() || !Known.isNonNegative())
+    return SSBits;
+  unsigned LHSSignBits = Known.countMinLeadingZeros();
 
   unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
-  unsigned DivBits = Num->getType()->getScalarSizeInBits() - SignBits;
-  if (IsSigned)
-    ++DivBits;
+  unsigned DivBits = SSBits - SignBits;
   return DivBits;
 }
 
@@ -1219,7 +1238,7 @@ Value *AMDGPUCodeGenPrepareImpl::expandDivRem24(IRBuilder<> &Builder,
   // If Num bits <= 24, assume 0 signbits.
   unsigned AtLeast = (SSBits <= 24) ? 0 : (SSBits - 24 + IsSigned);
   int DivBits = getDivNumBits(I, Num, Den, AtLeast, IsSigned);
-  if (DivBits == -1)
+  if (DivBits == -1 || DivBits > 24)
     return nullptr;
   return expandDivRem24Impl(Builder, I, Num, Den, DivBits, IsDiv, IsSigned);
 }

llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll

@@ -10694,3 +10694,194 @@ define <2 x i64> @v_udiv_i64_exact(<2 x i64> %num) {
    %result = udiv exact <2 x i64> %num, <i64 4096, i64 1024>
    ret <2 x i64> %result
 }
+
+define i64 @udiv_i64_gt_smax(i8 %size) {
+; GFX6-LABEL: udiv_i64_gt_smax:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    s_add_u32 s4, 0, 0x99986000
+; GFX6-NEXT:    v_mul_hi_u32 v2, s4, -10
+; GFX6-NEXT:    s_addc_u32 s5, 0, 0x59
+; GFX6-NEXT:    s_add_i32 s5, s5, 0x19999940
+; GFX6-NEXT:    s_mul_i32 s6, s4, -10
+; GFX6-NEXT:    v_subrev_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT:    s_mul_i32 s7, s5, -10
+; GFX6-NEXT:    v_mov_b32_e32 v3, s6
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s7, v2
+; GFX6-NEXT:    v_mul_hi_u32 v4, s5, v3
+; GFX6-NEXT:    v_mul_lo_u32 v5, s4, v2
+; GFX6-NEXT:    v_mul_hi_u32 v3, s4, v3
+; GFX6-NEXT:    v_mul_hi_u32 v6, s4, v2
+; GFX6-NEXT:    s_mul_i32 s6, s5, s6
+; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_mul_hi_u32 v6, s5, v2
+; GFX6-NEXT:    v_mul_lo_u32 v2, s5, v2
+; GFX6-NEXT:    v_add_i32_e32 v3, vcc, s6, v3
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v5, v4, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GFX6-NEXT:    v_mov_b32_e32 v4, s5
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, s4, v2
+; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GFX6-NEXT:    v_not_b32_e32 v0, v0
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, v4, v3, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v4, v0, v3
+; GFX6-NEXT:    v_mul_hi_u32 v5, v0, v2
+; GFX6-NEXT:    v_mul_hi_u32 v6, v0, v3
+; GFX6-NEXT:    v_not_b32_e32 v1, v1
+; GFX6-NEXT:    v_mul_hi_u32 v7, v1, v3
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GFX6-NEXT:    v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v6, v1, v2
+; GFX6-NEXT:    v_mul_hi_u32 v2, v1, v2
+; GFX6-NEXT:    v_mul_lo_u32 v3, v1, v3
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT:    v_addc_u32_e32 v2, vcc, v5, v2, vcc
+; GFX6-NEXT:    v_addc_u32_e32 v4, vcc, 0, v7, vcc
+; GFX6-NEXT:    v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT:    v_addc_u32_e32 v3, vcc, 0, v4, vcc
+; GFX6-NEXT:    v_mul_lo_u32 v4, v3, 10
+; GFX6-NEXT:    v_mul_hi_u32 v5, v2, 10
+; GFX6-NEXT:    v_mul_lo_u32 v6, v2, 10
+; GFX6-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GFX6-NEXT:    v_sub_i32_e32 v0, vcc, v0, v6
+; GFX6-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
+; GFX6-NEXT:    v_subrev_i32_e32 v4, vcc, 10, v0
+; GFX6-NEXT:    v_subbrev_u32_e32 v5, vcc, 0, v1, vcc
+; GFX6-NEXT:    v_cmp_lt_u32_e32 vcc, 9, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GFX6-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v5
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, -1, v4, vcc
+; GFX6-NEXT:    v_add_i32_e32 v5, vcc, 2, v2
+; GFX6-NEXT:    v_addc_u32_e32 v6, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_add_i32_e32 v7, vcc, 1, v2
+; GFX6-NEXT:    v_cmp_lt_u32_e64 s[4:5], 9, v0
+; GFX6-NEXT:    v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[4:5]
+; GFX6-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX6-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v4
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e32 v4, v7, v5, vcc
+; GFX6-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GFX6-NEXT:    v_cndmask_b32_e32 v1, v8, v6, vcc
+; GFX6-NEXT:    v_cndmask_b32_e64 v0, v2, v4, s[4:5]
+; GFX6-NEXT:    v_cndmask_b32_e64 v1, v3, v1, s[4:5]
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: udiv_i64_gt_smax:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_add_u32 s4, 0, 0x99986000
+; GFX9-NEXT:    s_addc_u32 s5, 0, 0x59
+; GFX9-NEXT:    s_add_i32 s5, s5, 0x19999940
+; GFX9-NEXT:    s_mul_hi_u32 s9, s4, -10
+; GFX9-NEXT:    s_sub_i32 s9, s9, s4
+; GFX9-NEXT:    s_mul_i32 s10, s5, -10
+; GFX9-NEXT:    s_mul_i32 s6, s4, -10
+; GFX9-NEXT:    s_add_i32 s9, s9, s10
+; GFX9-NEXT:    s_mul_hi_u32 s7, s5, s6
+; GFX9-NEXT:    s_mul_i32 s8, s5, s6
+; GFX9-NEXT:    s_mul_i32 s11, s4, s9
+; GFX9-NEXT:    s_mul_hi_u32 s6, s4, s6
+; GFX9-NEXT:    s_mul_hi_u32 s10, s4, s9
+; GFX9-NEXT:    s_add_u32 s6, s6, s11
+; GFX9-NEXT:    s_addc_u32 s10, 0, s10
+; GFX9-NEXT:    s_add_u32 s6, s6, s8
+; GFX9-NEXT:    s_mul_hi_u32 s11, s5, s9
+; GFX9-NEXT:    s_addc_u32 s6, s10, s7
+; GFX9-NEXT:    s_addc_u32 s7, s11, 0
+; GFX9-NEXT:    s_mul_i32 s8, s5, s9
+; GFX9-NEXT:    s_add_u32 s6, s6, s8
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    s_addc_u32 s7, 0, s7
+; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, s4, v1
+; GFX9-NEXT:    s_cmp_lg_u64 vcc, 0
+; GFX9-NEXT:    v_not_b32_sdwa v4, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; GFX9-NEXT:    s_addc_u32 s6, s5, s7
+; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v4, s6, 0
+; GFX9-NEXT:    v_mul_hi_u32 v6, v4, v5
+; GFX9-NEXT:    v_mov_b32_e32 v3, 31
+; GFX9-NEXT:    v_ashrrev_i32_sdwa v0, v3, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_not_b32_e32 v7, v0
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v7, v5, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v7, s6, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, v8, v1, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v0, v2
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v3, 10, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[4:5], v5, 10, v[1:2]
+; GFX9-NEXT:    v_sub_co_u32_e32 v0, vcc, v4, v0
+; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v7, v1, vcc
+; GFX9-NEXT:    v_subrev_co_u32_e32 v2, vcc, 10, v0
+; GFX9-NEXT:    v_subbrev_co_u32_e32 v4, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_cmp_lt_u32_e32 vcc, 9, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, -1, v2, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, 2, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, 1, v3
+; GFX9-NEXT:    v_cmp_lt_u32_e64 s[4:5], 9, v0
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, -1, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, -1, v0, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, v7, v4, vcc
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v8, v6, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v2, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v1, v5, v1, s[4:5]
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %esize = sext i8 %size to i64
+  %minus = sub nuw nsw i64 -1, %esize
+  %div = udiv i64 %minus, 10
+  ret i64 %div
+}
+
+define i64 @udiv_i64_9divbits(i8 %size) {
+; GFX6-LABEL: udiv_i64_9divbits:
+; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX6-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; GFX6-NEXT:    s_mov_b32 s4, 0x41200000
+; GFX6-NEXT:    v_mul_f32_e32 v1, 0x3dcccccd, v0
+; GFX6-NEXT:    v_trunc_f32_e32 v1, v1
+; GFX6-NEXT:    v_cvt_u32_f32_e32 v2, v1
+; GFX6-NEXT:    v_mad_f32 v0, -v1, s4, v0
+; GFX6-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s4
+; GFX6-NEXT:    v_mov_b32_e32 v1, 0
+; GFX6-NEXT:    v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT:    v_and_b32_e32 v0, 0x1ff, v0
+; GFX6-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: udiv_i64_9divbits:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, 1
+; GFX9-NEXT:    v_add_u32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, v0
+; GFX9-NEXT:    s_mov_b32 s4, 0x41200000
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x3dcccccd, v0
+; GFX9-NEXT:    v_trunc_f32_e32 v1, v1
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v2, v1
+; GFX9-NEXT:    v_mad_f32 v0, -v1, s4, v0
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v0|, s4
+; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v0, vcc, 0, v2, vcc
+; GFX9-NEXT:    v_and_b32_e32 v0, 0x1ff, v0
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %zextend = zext i8 %size to i64
+  %num = add nuw nsw i64 1, %zextend
+  %div = udiv i64 %num, 10
+  ret i64 %div
+}

llvm/test/CodeGen/AMDGPU/bypass-div.ll

@@ -1021,8 +1021,116 @@ define i64 @sdiv64_known32(i64 %a, i64 %b) {
 ; GFX9-LABEL: sdiv64_known32:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NEXT:    v_ashrrev_i32_e32 v0, 31, v3
+; GFX9-NEXT:    v_or_b32_e32 v5, v2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v4, 0
+; GFX9-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v7, v1
+; GFX9-NEXT:    v_mov_b32_e32 v6, v3
+; GFX9-NEXT:    ; implicit-def: $vgpr4_vgpr5
+; GFX9-NEXT:    s_and_saveexec_b64 s[4:5], vcc
+; GFX9-NEXT:    s_xor_b64 s[6:7], exec, s[4:5]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_2
+; GFX9-NEXT:  ; %bb.1:
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v1, v6
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v3, v0
+; GFX9-NEXT:    v_sub_co_u32_e32 v11, vcc, 0, v6
+; GFX9-NEXT:    v_subb_co_u32_e32 v12, vcc, 0, v0, vcc
+; GFX9-NEXT:    v_madmk_f32 v1, v3, 0x4f800000, v1
+; GFX9-NEXT:    v_rcp_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_f32_e32 v1, 0x5f7ffffc, v1
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x2f800000, v1
+; GFX9-NEXT:    v_trunc_f32_e32 v3, v3
+; GFX9-NEXT:    v_madmk_f32 v1, v3, 0xcf800000, v1
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v10, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v1
+; GFX9-NEXT:    v_mul_lo_u32 v5, v11, v10
+; GFX9-NEXT:    v_mul_lo_u32 v8, v12, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v11, v1, 0
+; GFX9-NEXT:    v_add3_u32 v8, v4, v5, v8
+; GFX9-NEXT:    v_mul_hi_u32 v9, v1, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v1, v8, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v13, vcc, v9, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v10, v3, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v10, v8, 0
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v13, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v5, v4, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, v10, v4, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v5, v11, v13
+; GFX9-NEXT:    v_mul_lo_u32 v8, v12, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v11, v1, 0
+; GFX9-NEXT:    v_add3_u32 v8, v4, v5, v8
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[4:5], v13, v8, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v1, v8, 0
+; GFX9-NEXT:    v_mul_hi_u32 v12, v1, v3
+; GFX9-NEXT:    v_mad_u64_u32 v[10:11], s[4:5], v13, v3, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v12, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v8, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v11, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v5, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v13, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v7, v5, 0
+; GFX9-NEXT:    v_mul_hi_u32 v8, v7, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v8, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v2, v1, 0
+; GFX9-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v2, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v10, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, v11, v4, vcc
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v9, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v8
+; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v8, v0, v1
+; GFX9-NEXT:    v_mul_lo_u32 v9, v6, v5
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[4:5], v6, v1, 0
+; GFX9-NEXT:    v_add3_u32 v4, v4, v9, v8
+; GFX9-NEXT:    v_sub_u32_e32 v8, v2, v4
+; GFX9-NEXT:    v_sub_co_u32_e32 v3, vcc, v7, v3
+; GFX9-NEXT:    v_subb_co_u32_e64 v7, s[4:5], v8, v0, vcc
+; GFX9-NEXT:    v_sub_co_u32_e64 v8, s[4:5], v3, v6
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v7, s[4:5], 0, v7, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v7, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, -1, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e64 s[4:5], v8, v6
+; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, -1, s[4:5]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[4:5], v7, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v8, s[4:5]
+; GFX9-NEXT:    v_add_co_u32_e64 v8, s[4:5], 2, v1
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v2, v4, vcc
+; GFX9-NEXT:    v_addc_co_u32_e64 v9, s[4:5], 0, v5, s[4:5]
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v2, v0
+; GFX9-NEXT:    v_add_co_u32_e64 v10, s[4:5], 1, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v3, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v11, s[4:5], 0, v5, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v0
+; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], 0, v7
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v4, v3, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v9, s[4:5]
+; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v0
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v10, v8, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v1, v0, vcc
+; GFX9-NEXT:    ; implicit-def: $vgpr2_vgpr3
+; GFX9-NEXT:    ; implicit-def: $vgpr0_vgpr1
+; GFX9-NEXT:  .LBB10_2: ; %Flow
+; GFX9-NEXT:    s_andn2_saveexec_b64 s[4:5], s[6:7]
+; GFX9-NEXT:    s_cbranch_execz .LBB10_4
+; GFX9-NEXT:  ; %bb.3:
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v2, 0, v3
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
@@ -1033,14 +1141,17 @@ define i64 @sdiv64_known32(i64 %a, i64 %b) {
 ; GFX9-NEXT:    v_mul_lo_u32 v2, v0, v3
 ; GFX9-NEXT:    v_add_u32_e32 v4, 1, v0
 ; GFX9-NEXT:    v_sub_u32_e32 v1, v1, v2
-; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
 ; GFX9-NEXT:    v_sub_u32_e32 v2, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
+; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v4, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v2, 1, v0
 ; GFX9-NEXT:    v_cmp_ge_u32_e32 vcc, v1, v3
-; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v1, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v0, v2, vcc
+; GFX9-NEXT:  .LBB10_4:
+; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT:    v_mov_b32_e32 v0, v4
+; GFX9-NEXT:    v_mov_b32_e32 v1, v5
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
   %a.ext = ashr i64 %a, 32
   %b.ext = ashr i64 %b, 32