PeepholeOpt: Do not add subregister indexes to reg_sequence operands

Given the rest of the pass just gives up when it needs to compose
subregisters, folding a subregister extract directly into a reg_sequence
is counterproductive. Later fold attempts in the function will give up
on the subregister operand, preventing looking up through the reg_sequence.

It may still be profitable to do these folds if we start handling
the composes. There are some test regressions, but this mostly
looks better.

Author: arsenm

llvm/lib/CodeGen/PeepholeOptimizer.cpp

@@ -436,6 +436,12 @@ class RegSequenceRewriter : public Rewriter {
     if ((CurrentSrcIdx & 1) != 1 || CurrentSrcIdx > CopyLike.getNumOperands())
       return false;
 
+    // Do not introduce new subregister uses in a reg_sequence. Until composing
+    // subregister indices is supported while folding, we're just blocking
+    // folding of subregister copies later in the function.
+    if (NewSubReg)
+      return false;
+
     MachineOperand &MO = CopyLike.getOperand(CurrentSrcIdx);
     MO.setReg(NewReg);
     MO.setSubReg(NewSubReg);

llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll

@@ -1667,7 +1667,7 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX9-NEXT:    v_mul_lo_u32 v3, s10, v1
 ; GFX9-NEXT:    v_mul_hi_u32 v4, s10, v0
 ; GFX9-NEXT:    v_mul_hi_u32 v0, s11, v0
-; GFX9-NEXT:    v_mul_hi_u32 v6, s11, v1
+; GFX9-NEXT:    v_mul_hi_u32 v5, s11, v1
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v3
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
@@ -1679,155 +1679,155 @@ define amdgpu_kernel void @sdivrem_v2i64(ptr addrspace(1) %out0, ptr addrspace(1
 ; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v3
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v5, vcc, v0, v2
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s8, v5, 0
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v0, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s8, v6, 0
 ; GFX9-NEXT:    v_cndmask_b32_e64 v0, 0, 1, vcc
 ; GFX9-NEXT:    v_add_u32_e32 v3, v4, v3
-; GFX9-NEXT:    v_add3_u32 v4, v3, v0, v6
+; GFX9-NEXT:    v_add3_u32 v8, v3, v0, v5
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v2
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s8, v4, v[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v6, s11
-; GFX9-NEXT:    v_sub_co_u32_e32 v8, vcc, s10, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s9, v5, v[2:3]
+; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s8, v8, v[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v5, s11
+; GFX9-NEXT:    v_sub_co_u32_e32 v1, vcc, s10, v1
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s9, v6, v[2:3]
 ; GFX9-NEXT:    s_ashr_i32 s10, s3, 31
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_subb_co_u32_e64 v6, s[0:1], v6, v2, vcc
-; GFX9-NEXT:    v_sub_u32_e32 v1, s11, v2
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[0:1]
-; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v8
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v7, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, -1, s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v6
-; GFX9-NEXT:    v_subrev_co_u32_e32 v10, vcc, s8, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v2, v3, s[0:1]
-; GFX9-NEXT:    v_subbrev_co_u32_e64 v11, s[0:1], 0, v1, vcc
-; GFX9-NEXT:    v_add_co_u32_e64 v3, s[0:1], 1, v5
-; GFX9-NEXT:    v_addc_co_u32_e64 v12, s[0:1], 0, v4, s[0:1]
+; GFX9-NEXT:    v_subb_co_u32_e64 v2, s[0:1], v5, v3, vcc
+; GFX9-NEXT:    v_sub_u32_e32 v3, s11, v3
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
+; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v1
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, -1, s[0:1]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v2
+; GFX9-NEXT:    v_subrev_co_u32_e32 v10, vcc, s8, v1
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, v4, v5, s[0:1]
+; GFX9-NEXT:    v_subbrev_co_u32_e64 v11, s[0:1], 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e64 v5, s[0:1], 1, v6
+; GFX9-NEXT:    v_addc_co_u32_e64 v12, s[0:1], 0, v8, s[0:1]
 ; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s9, v11
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, -1, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, -1, s[0:1]
 ; GFX9-NEXT:    v_cmp_le_u32_e64 s[0:1], s8, v10
 ; GFX9-NEXT:    v_cndmask_b32_e64 v13, 0, -1, s[0:1]
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s9, v11
-; GFX9-NEXT:    v_cndmask_b32_e64 v13, v2, v13, s[0:1]
-; GFX9-NEXT:    v_add_co_u32_e64 v14, s[0:1], 1, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v13, v4, v13, s[0:1]
+; GFX9-NEXT:    v_add_co_u32_e64 v14, s[0:1], 1, v5
 ; GFX9-NEXT:    v_addc_co_u32_e64 v15, s[0:1], 0, v12, s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s18, s6
 ; GFX9-NEXT:    s_addc_u32 s1, s19, s6
 ; GFX9-NEXT:    s_add_u32 s2, s2, s10
 ; GFX9-NEXT:    s_mov_b32 s11, s10
 ; GFX9-NEXT:    s_addc_u32 s3, s3, s10
 ; GFX9-NEXT:    s_xor_b64 s[2:3], s[2:3], s[10:11]
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v2, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v4, s3
 ; GFX9-NEXT:    v_cvt_f32_u32_e32 v16, s2
-; GFX9-NEXT:    v_subb_co_u32_e32 v1, vcc, v1, v7, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0x4f800000, v2
-; GFX9-NEXT:    v_add_f32_e32 v2, v2, v16
-; GFX9-NEXT:    v_rcp_iflag_f32_e32 v2, v2
+; GFX9-NEXT:    v_subb_co_u32_e32 v3, vcc, v3, v7, vcc
+; GFX9-NEXT:    v_mul_f32_e32 v4, 0x4f800000, v4
+; GFX9-NEXT:    v_add_f32_e32 v4, v4, v16
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v4, v4
 ; GFX9-NEXT:    v_subrev_co_u32_e32 v7, vcc, s8, v10
-; GFX9-NEXT:    v_subbrev_co_u32_e32 v16, vcc, 0, v1, vcc
-; GFX9-NEXT:    v_mul_f32_e32 v1, 0x5f7ffffc, v2
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0x2f800000, v1
-; GFX9-NEXT:    v_trunc_f32_e32 v17, v2
-; GFX9-NEXT:    v_mul_f32_e32 v2, 0xcf800000, v17
-; GFX9-NEXT:    v_add_f32_e32 v1, v2, v1
-; GFX9-NEXT:    v_cvt_u32_f32_e32 v18, v1
+; GFX9-NEXT:    v_subbrev_co_u32_e32 v16, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_mul_f32_e32 v3, 0x5f7ffffc, v4
+; GFX9-NEXT:    v_mul_f32_e32 v4, 0x2f800000, v3
+; GFX9-NEXT:    v_trunc_f32_e32 v17, v4
+; GFX9-NEXT:    v_mul_f32_e32 v4, 0xcf800000, v17
+; GFX9-NEXT:    v_add_f32_e32 v3, v4, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v18, v3
 ; GFX9-NEXT:    s_xor_b64 s[8:9], s[0:1], s[6:7]
 ; GFX9-NEXT:    s_sub_u32 s5, 0, s2
 ; GFX9-NEXT:    v_cmp_ne_u32_e32 vcc, 0, v13
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s5, v18, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v13, v3, v14, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s5, v18, 0
+; GFX9-NEXT:    v_cndmask_b32_e32 v13, v5, v14, vcc
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v14, v17
 ; GFX9-NEXT:    s_subb_u32 s20, 0, s3
 ; GFX9-NEXT:    v_cndmask_b32_e32 v12, v12, v15, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e32 v7, v10, v7, vcc
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], s5, v14, v[2:3]
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], s5, v14, v[4:5]
 ; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], 0, v9
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, v4, v12, s[0:1]
-; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[18:19], s20, v18, v[2:3]
-; GFX9-NEXT:    v_mul_lo_u32 v3, v14, v1
 ; GFX9-NEXT:    v_cndmask_b32_e32 v10, v11, v16, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v4, v18, v2
-; GFX9-NEXT:    v_mul_hi_u32 v11, v18, v1
-; GFX9-NEXT:    v_mul_hi_u32 v1, v14, v1
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v5, v13, s[0:1]
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v11
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v11, v14, v2
-; GFX9-NEXT:    v_add_u32_e32 v3, v4, v3
-; GFX9-NEXT:    v_mul_hi_u32 v4, v18, v2
-; GFX9-NEXT:    v_mul_hi_u32 v2, v14, v2
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v11, v1
-; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v1, vcc, v1, v3
-; GFX9-NEXT:    v_add_u32_e32 v4, v11, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v18, v1
-; GFX9-NEXT:    v_add3_u32 v2, v4, v3, v2
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[18:19], s5, v11, 0
-; GFX9-NEXT:    v_addc_co_u32_e32 v12, vcc, v14, v2, vcc
-; GFX9-NEXT:    v_mov_b32_e32 v1, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v8, v7, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s[0:1]
-; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s5, v12, v[1:2]
-; GFX9-NEXT:    v_xor_b32_e32 v8, s16, v5
-; GFX9-NEXT:    v_xor_b32_e32 v9, s17, v9
-; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], s20, v11, v[1:2]
-; GFX9-NEXT:    v_mov_b32_e32 v10, s17
-; GFX9-NEXT:    v_subrev_co_u32_e32 v1, vcc, s16, v8
-; GFX9-NEXT:    v_xor_b32_e32 v5, s4, v7
-; GFX9-NEXT:    v_mul_lo_u32 v7, v12, v3
-; GFX9-NEXT:    v_mul_lo_u32 v8, v11, v4
-; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v9, v10, vcc
-; GFX9-NEXT:    v_mul_hi_u32 v9, v11, v3
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[18:19], s20, v18, v[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v12, s[0:1]
+; GFX9-NEXT:    v_mul_lo_u32 v8, v14, v3
+; GFX9-NEXT:    v_mul_lo_u32 v9, v18, v4
+; GFX9-NEXT:    v_mul_hi_u32 v11, v18, v3
+; GFX9-NEXT:    v_mul_hi_u32 v3, v14, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v1, v7, s[0:1]
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v9
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v8, v11
 ; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v9
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v9, v12, v4
-; GFX9-NEXT:    v_mul_hi_u32 v3, v12, v3
-; GFX9-NEXT:    v_add_u32_e32 v7, v8, v7
-; GFX9-NEXT:    v_mul_hi_u32 v8, v11, v4
-; GFX9-NEXT:    v_mul_hi_u32 v4, v12, v4
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v9, v3
+; GFX9-NEXT:    v_mul_lo_u32 v11, v14, v4
+; GFX9-NEXT:    v_add_u32_e32 v8, v9, v8
+; GFX9-NEXT:    v_mul_hi_u32 v9, v18, v4
+; GFX9-NEXT:    v_mul_hi_u32 v4, v14, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v11, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v9
 ; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v8
+; GFX9-NEXT:    v_add_u32_e32 v9, v11, v9
 ; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v7
-; GFX9-NEXT:    v_add_u32_e32 v8, v9, v8
+; GFX9-NEXT:    v_add3_u32 v4, v9, v8, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v18, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v14, v4, vcc
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[18:19], s5, v8, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v6, v13, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v10, v2, v10, s[0:1]
+; GFX9-NEXT:    v_mov_b32_e32 v1, v4
+; GFX9-NEXT:    v_mad_u64_u32 v[1:2], s[0:1], s5, v9, v[1:2]
+; GFX9-NEXT:    v_xor_b32_e32 v11, s17, v5
+; GFX9-NEXT:    v_xor_b32_e32 v6, s16, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[4:5], s[0:1], s20, v8, v[1:2]
+; GFX9-NEXT:    v_mov_b32_e32 v12, s17
+; GFX9-NEXT:    v_subrev_co_u32_e32 v1, vcc, s16, v6
+; GFX9-NEXT:    v_xor_b32_e32 v5, s4, v7
+; GFX9-NEXT:    v_mul_lo_u32 v6, v9, v3
+; GFX9-NEXT:    v_mul_lo_u32 v7, v8, v4
+; GFX9-NEXT:    v_subb_co_u32_e32 v2, vcc, v11, v12, vcc
+; GFX9-NEXT:    v_mul_hi_u32 v11, v8, v3
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v7
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-NEXT:    v_add3_u32 v4, v8, v7, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v11
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v11, v9, v4
+; GFX9-NEXT:    v_mul_hi_u32 v3, v9, v3
+; GFX9-NEXT:    v_add_u32_e32 v6, v7, v6
+; GFX9-NEXT:    v_mul_hi_u32 v7, v8, v4
+; GFX9-NEXT:    v_mul_hi_u32 v4, v9, v4
 ; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v11, v3
-; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v12, v4, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v7, s9, v3
-; GFX9-NEXT:    v_mul_lo_u32 v8, s8, v4
-; GFX9-NEXT:    v_mul_hi_u32 v10, s8, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v11, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v7
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v6
+; GFX9-NEXT:    v_add_u32_e32 v7, v11, v7
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_add3_u32 v4, v7, v6, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v8, v3
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, v9, v4, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v6, s9, v3
+; GFX9-NEXT:    v_mul_lo_u32 v7, s8, v4
+; GFX9-NEXT:    v_mul_hi_u32 v9, s8, v3
 ; GFX9-NEXT:    v_mul_hi_u32 v3, s9, v3
-; GFX9-NEXT:    v_mul_hi_u32 v12, s9, v4
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v7, vcc, v7, v10
+; GFX9-NEXT:    v_mul_hi_u32 v13, s9, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v7
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-NEXT:    v_mul_lo_u32 v10, s9, v4
-; GFX9-NEXT:    v_add_u32_e32 v7, v8, v7
-; GFX9-NEXT:    v_mul_hi_u32 v8, s8, v4
-; GFX9-NEXT:    v_xor_b32_e32 v6, s4, v6
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v10, v3
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v3, v7
-; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s2, v11, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, s4
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v6, v9
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
+; GFX9-NEXT:    v_mul_lo_u32 v9, s9, v4
+; GFX9-NEXT:    v_add_u32_e32 v6, v7, v6
+; GFX9-NEXT:    v_mul_hi_u32 v7, s8, v4
+; GFX9-NEXT:    v_xor_b32_e32 v10, s4, v10
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v9, v3
+; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v3, vcc, v3, v7
 ; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v11, vcc, v3, v6
+; GFX9-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], s2, v11, 0
+; GFX9-NEXT:    v_cndmask_b32_e64 v12, 0, 1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v7, v9, v7
+; GFX9-NEXT:    v_mov_b32_e32 v8, s4
 ; GFX9-NEXT:    v_subrev_co_u32_e32 v5, vcc, s4, v5
-; GFX9-NEXT:    v_add_u32_e32 v8, v10, v8
-; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v6, v9, vcc
-; GFX9-NEXT:    v_add3_u32 v9, v8, v7, v12
+; GFX9-NEXT:    v_add3_u32 v9, v7, v12, v13
+; GFX9-NEXT:    v_subb_co_u32_e32 v6, vcc, v10, v8, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[7:8], s[0:1], s2, v9, v[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v10, s9
 ; GFX9-NEXT:    v_sub_co_u32_e32 v3, vcc, s8, v3

llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll

@@ -10041,11 +10041,9 @@ define i64 @udiv_i64_gt_smax(i8 %size) {
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    s_mov_b32 s6, 0xcccccccc
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[4:5], v5, s4, v[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v6, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, v1
-; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, s6, v[2:3]
-; GFX9-NEXT:    v_mov_b32_e32 v0, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v6, v0
+; GFX9-NEXT:    v_mov_b32_e32 v0, v2
+; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v4, s6, v[0:1]
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v3, v1
 ; GFX9-NEXT:    v_addc_co_u32_e64 v1, s[4:5], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[4:5], v5, s6, v[0:1]
 ; GFX9-NEXT:    v_alignbit_b32 v0, v1, v0, 3