CodeGen using True16 D16 LDS ld/st pseudo instructions

Implement new pseudos with the suffix _t16 which have VGPR_16 as the
store src or load dst. This affects LDS 8 and 16-bit loads and stores.
Lower the pseudos to the existing real instructions in MC inst layer
with VGPR_32 src or dst (which makes them consistent with the hardware
encoding). This patch reduces VGPR usage by making hi halves of VGPRs
available for other values.
Modified lit tests.

Author: abhigargrepo

llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp

@@ -187,6 +187,47 @@ void AMDGPUMCInstLower::lower(const MachineInstr *MI, MCInst &OutMI) const {
     OutMI.addOperand(Dest);
     OutMI.addOperand(Src);
     return;
+  } else if (const auto *Info = AMDGPU::getT16D16Helper(Opcode)) {
+    uint16_t OpName = AMDGPU::OpName::OPERAND_LAST;
+    if (TII->isDS(Opcode)) {
+      if (MI->mayLoad())
+        OpName = llvm::AMDGPU::OpName::vdst;
+      else if (MI->mayStore())
+        OpName = llvm::AMDGPU::OpName::data0;
+      else
+        llvm_unreachable("LDS load or store expected");
+    } else {
+      OpName = AMDGPU::hasNamedOperand(Opcode, llvm::AMDGPU::OpName::vdata)
+                   ? llvm::AMDGPU::OpName::vdata
+                   : llvm::AMDGPU::OpName::vdst;
+    }
+    int VDstOrVDataIdx = AMDGPU::getNamedOperandIdx(Opcode, OpName);
+    MachineOperand MIVDstOrVData = MI->getOperand(VDstOrVDataIdx);
+    bool IsHi = AMDGPU::isHi16Reg(MIVDstOrVData.getReg(), TRI);
+    Opcode = IsHi ? Info->HiOp : Info->LoOp;
+    MIVDstOrVData.clearParent(); // Avoid use list error in setReg call
+    MIVDstOrVData.setReg(TRI.get32BitRegister(MIVDstOrVData.getReg()));
+
+    int MCOpcode = TII->pseudoToMCOpcode(Opcode);
+    assert(MCOpcode != -1 &&
+           "Pseudo instruction doesn't have a target-specific version");
+    OutMI.setOpcode(MCOpcode);
+    for (int I = 0, E = MI->getNumExplicitOperands(); I < E; I++) {
+      const MachineOperand &MO = MI->getOperand(I);
+      MCOperand MCOp;
+      if (I == VDstOrVDataIdx)
+        lowerOperand(MIVDstOrVData, MCOp);
+      else
+        lowerOperand(MO, MCOp);
+      OutMI.addOperand(MCOp);
+    }
+
+    if (AMDGPU::hasNamedOperand(MCOpcode, AMDGPU::OpName::vdst_in)) {
+      MCOperand MCOp;
+      lowerOperand(MIVDstOrVData, MCOp);
+      OutMI.addOperand(MCOp);
+    }
+    return;
   } else if (Opcode == AMDGPU::SI_TCRETURN ||
              Opcode == AMDGPU::SI_TCRETURN_GFX) {
     // TODO: How to use branch immediate and avoid register+add?

llvm/lib/Target/AMDGPU/DSInstructions.td

@@ -127,6 +127,15 @@ multiclass DS_1A1D_NORET_mc<string opName, RegisterClass rc = VGPR_32> {
   }
 }
 
+multiclass DS_1A1D_NORET_t16<string opName, RegisterClass rc = VGPR_32> 
+: DS_1A1D_NORET_mc<opName, rc> {
+  let has_m0_read = 0 in {
+    let True16Predicate = UseRealTrue16Insts in {
+      def "_t16" : DS_1A1D_NORET<opName#"_t16", VGPR_16>, True16D16Table<NAME#"_D16_HI", NAME>;
+    }
+  }
+}
+
 multiclass DS_1A1D_NORET_mc_gfx9<string opName, RegisterClass rc = VGPR_32> {
   let has_m0_read = 0 in {
     def "" : DS_1A1D_NORET<opName, rc>;
@@ -294,6 +303,15 @@ multiclass DS_1A_RET_mc<string opName, RegisterClass rc = VGPR_32, bit HasTiedOu
   }
 }
 
+multiclass DS_1A_RET_t16<string opName, RegisterClass rc = VGPR_32, bit HasTiedOutput = 0, Operand ofs = Offset> 
+: DS_1A_RET_mc<opName, rc, HasTiedOutput, ofs> {
+  let has_m0_read = 0 in {
+    let True16Predicate = UseRealTrue16Insts in {
+      def "_t16" : DS_1A_RET<opName#"_t16", VGPR_16, HasTiedOutput, ofs>, True16D16Table<NAME#"_D16_HI", NAME#"_D16">;
+    }
+  }
+}
+
 multiclass DS_1A_RET_NoM0<string opName, RegisterClass rc = VGPR_32> {
   let has_m0_read = 0 in {
     def "" : DS_1A_RET<opName, rc>;
@@ -457,8 +475,6 @@ defm DS_MIN_F32       : DS_1A1D_NORET_mc<"ds_min_f32">;
 defm DS_MAX_F32       : DS_1A1D_NORET_mc<"ds_max_f32">;
 
 let mayLoad = 0 in {
-defm DS_WRITE_B8      : DS_1A1D_NORET_mc<"ds_write_b8">;
-defm DS_WRITE_B16     : DS_1A1D_NORET_mc<"ds_write_b16">;
 defm DS_WRITE_B32     : DS_1A1D_NORET_mc<"ds_write_b32">;
 defm DS_WRITE2_B32    : DS_1A2D_Off8_NORET_mc<"ds_write2_b32">;
 defm DS_WRITE2ST64_B32: DS_1A2D_Off8_NORET_mc<"ds_write2st64_b32">;
@@ -473,6 +489,9 @@ def DS_WRITE_B16_D16_HI : DS_1A1D_NORET<"ds_write_b16_d16_hi">;
 
 } // End has_m0_read = 0
 
+defm DS_WRITE_B8      : DS_1A1D_NORET_t16<"ds_write_b8">;
+defm DS_WRITE_B16     : DS_1A1D_NORET_t16<"ds_write_b16">;
+
 let SubtargetPredicate = HasDSAddTid in {
 def DS_WRITE_ADDTID_B32 : DS_0A1D_NORET<"ds_write_addtid_b32">;
 }
@@ -625,10 +644,7 @@ def DS_SWIZZLE_B32 : DS_1A_RET <"ds_swizzle_b32", VGPR_32, 0, Swizzle>;
 }
 
 let mayStore = 0 in {
-defm DS_READ_I8      : DS_1A_RET_mc<"ds_read_i8">;
-defm DS_READ_U8      : DS_1A_RET_mc<"ds_read_u8">;
 defm DS_READ_I16     : DS_1A_RET_mc<"ds_read_i16">;
-defm DS_READ_U16     : DS_1A_RET_mc<"ds_read_u16">;
 defm DS_READ_B32     : DS_1A_RET_mc<"ds_read_b32">;
 defm DS_READ_B64     : DS_1A_RET_mc<"ds_read_b64", VReg_64>;
 
@@ -649,6 +665,10 @@ def DS_READ_U16_D16_HI : DS_1A_RET_Tied<"ds_read_u16_d16_hi">;
 }
 } // End has_m0_read = 0
 
+defm DS_READ_I8      : DS_1A_RET_t16<"ds_read_i8">;
+defm DS_READ_U8      : DS_1A_RET_t16<"ds_read_u8">;
+defm DS_READ_U16     : DS_1A_RET_t16<"ds_read_u16">;
+
 let SubtargetPredicate = HasDSAddTid in {
 def DS_READ_ADDTID_B32 : DS_0A_RET<"ds_read_addtid_b32">;
 }
@@ -784,34 +804,51 @@ multiclass DSReadPat_mc<DS_Pseudo inst, ValueType vt, string frag> {
   }
 }
 
+multiclass DSReadPat_t16<DS_Pseudo inst, ValueType vt, string frag> {
+
+  let OtherPredicates = [LDSRequiresM0Init] in {
+    def : DSReadPat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
+  }
+
+  let OtherPredicates = [NotLDSRequiresM0Init] in {
+    foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+      let True16Predicate = p in {
+        def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
+      }
+    let True16Predicate = UseRealTrue16Insts in {
+      def : DSReadPat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
+    }
+  }
+}
+
 class DSReadPat_D16 <DS_Pseudo inst, PatFrag frag, ValueType vt> : GCNPat <
   (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$in),
   (inst $ptr, Offset:$offset, (i1 0), $in)
 >;
 
 defm : DSReadPat_mc <DS_READ_I8, i32, "sextloadi8_local">;
-defm : DSReadPat_mc <DS_READ_I8,  i16, "sextloadi8_local">;
 defm : DSReadPat_mc <DS_READ_U8,  i32, "extloadi8_local">;
 defm : DSReadPat_mc <DS_READ_U8,  i32, "zextloadi8_local">;
-defm : DSReadPat_mc <DS_READ_U8,  i16, "extloadi8_local">;
-defm : DSReadPat_mc <DS_READ_U8,  i16, "zextloadi8_local">;
 defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
 defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
-defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
+defm : DSReadPat_t16 <DS_READ_I8,  i16, "sextloadi8_local">;
+defm : DSReadPat_t16 <DS_READ_U8,  i16, "extloadi8_local">;
+defm : DSReadPat_t16 <DS_READ_U8,  i16, "zextloadi8_local">;
+defm : DSReadPat_t16 <DS_READ_U16, i16, "load_local">;
 
 foreach vt = Reg32Types.types in {
 defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
 }
 
-defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_8_local">;
+defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_8_local">;
 defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_8_local">;
-defm : DSReadPat_mc <DS_READ_U8, i16, "atomic_load_zext_8_local">;
+defm : DSReadPat_t16 <DS_READ_U8, i16, "atomic_load_zext_8_local">;
 defm : DSReadPat_mc <DS_READ_U8, i32, "atomic_load_zext_8_local">;
-defm : DSReadPat_mc <DS_READ_I8, i16, "atomic_load_sext_8_local">;
+defm : DSReadPat_t16 <DS_READ_I8, i16, "atomic_load_sext_8_local">;
 defm : DSReadPat_mc <DS_READ_I8, i32, "atomic_load_sext_8_local">;
-defm : DSReadPat_mc <DS_READ_U16, i16, "atomic_load_16_local">;
+defm : DSReadPat_t16 <DS_READ_U16, i16, "atomic_load_16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "atomic_load_zext_16_local">;
 defm : DSReadPat_mc <DS_READ_I16, i32, "atomic_load_sext_16_local">;
@@ -850,18 +887,34 @@ multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
   }
 }
 
+multiclass DSWritePat_t16 <DS_Pseudo inst, ValueType vt, string frag> {
+  let OtherPredicates = [LDSRequiresM0Init] in {
+    def : DSWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
+  }
+
+  let OtherPredicates = [NotLDSRequiresM0Init] in {
+    foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+      let True16Predicate = p in {
+        def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
+      }
+    let True16Predicate = UseRealTrue16Insts in {
+      def : DSWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_t16"), vt, !cast<PatFrag>(frag)>;
+    }
+  }
+}
+
 defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
-defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
-defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
+defm : DSWritePat_t16 <DS_WRITE_B8, i16, "truncstorei8_local">;
+defm : DSWritePat_t16 <DS_WRITE_B16, i16, "store_local">;
 
 foreach vt = Reg32Types.types in {
 defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
 }
 
-defm : DSWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
+defm : DSWritePat_t16 <DS_WRITE_B8, i16, "atomic_store_8_local">;
 defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
-defm : DSWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
+defm : DSWritePat_t16 <DS_WRITE_B16, i16, "atomic_store_16_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
 defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
 defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;

llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll

@@ -239,48 +239,53 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_and_b32 s6, 0xffff, s0
 ; GFX11-NEXT:    s_lshr_b32 s5, s0, 16
-; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT:    v_mov_b16_e32 v0.l, s0
+; GFX11-NEXT:    v_mov_b32_e32 v5, s4
 ; GFX11-NEXT:    s_lshr_b32 s0, s1, 16
 ; GFX11-NEXT:    s_and_b32 s4, 0xffff, s1
-; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT:    v_mov_b16_e32 v0.h, s1
 ; GFX11-NEXT:    s_lshr_b32 s1, s2, 16
 ; GFX11-NEXT:    s_and_b32 s7, 0xffff, s2
+; GFX11-NEXT:    v_mov_b16_e32 v1.l, s2
 ; GFX11-NEXT:    s_lshr_b32 s2, s6, 8
 ; GFX11-NEXT:    s_lshr_b32 s6, s5, 8
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v7, s6
-; GFX11-NEXT:    v_dual_mov_b32 v4, s5 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT:    v_mov_b16_e32 v2.h, s2
+; GFX11-NEXT:    v_mov_b16_e32 v1.h, s5
 ; GFX11-NEXT:    s_lshr_b32 s4, s4, 8
 ; GFX11-NEXT:    s_lshr_b32 s5, s0, 8
+; GFX11-NEXT:    v_mov_b16_e32 v2.l, s0
 ; GFX11-NEXT:    s_lshr_b32 s0, s7, 8
-; GFX11-NEXT:    v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v9, s5
-; GFX11-NEXT:    ds_store_b8 v1, v0
-; GFX11-NEXT:    ds_store_b8 v1, v6 offset:1
-; GFX11-NEXT:    ds_store_b8 v1, v4 offset:2
-; GFX11-NEXT:    ds_store_b8 v1, v7 offset:3
-; GFX11-NEXT:    ds_store_b8 v1, v2 offset:4
-; GFX11-NEXT:    ds_store_b8 v1, v8 offset:5
-; GFX11-NEXT:    ds_store_b8 v1, v5 offset:6
-; GFX11-NEXT:    ds_store_b8 v1, v9 offset:7
-; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v5, s3
+; GFX11-NEXT:    v_mov_b16_e32 v3.l, s6
+; GFX11-NEXT:    v_mov_b16_e32 v3.h, s4
+; GFX11-NEXT:    v_mov_b16_e32 v4.l, s5
+; GFX11-NEXT:    ds_store_b8 v5, v0
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v2 offset:1
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v1 offset:2
+; GFX11-NEXT:    ds_store_b8 v5, v3 offset:3
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v0 offset:4
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v3 offset:5
+; GFX11-NEXT:    ds_store_b8 v5, v2 offset:6
+; GFX11-NEXT:    ds_store_b8 v5, v4 offset:7
+; GFX11-NEXT:    v_mov_b16_e32 v0.l, s0
 ; GFX11-NEXT:    s_lshr_b32 s0, s1, 8
-; GFX11-NEXT:    v_mov_b32_e32 v2, s1
-; GFX11-NEXT:    v_mov_b32_e32 v4, s0
+; GFX11-NEXT:    v_mov_b16_e32 v0.h, s1
+; GFX11-NEXT:    v_mov_b16_e32 v1.h, s0
 ; GFX11-NEXT:    s_and_b32 s0, 0xffff, s3
 ; GFX11-NEXT:    s_lshr_b32 s1, s3, 16
 ; GFX11-NEXT:    s_lshr_b32 s0, s0, 8
-; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    v_dual_mov_b32 v7, s1 :: v_dual_mov_b32 v6, s0
+; GFX11-NEXT:    v_mov_b16_e32 v2.l, s3
+; GFX11-NEXT:    v_mov_b16_e32 v2.h, s0
 ; GFX11-NEXT:    s_lshr_b32 s0, s1, 8
-; GFX11-NEXT:    v_mov_b32_e32 v8, s0
-; GFX11-NEXT:    ds_store_b8 v1, v3 offset:8
-; GFX11-NEXT:    ds_store_b8 v1, v0 offset:9
-; GFX11-NEXT:    ds_store_b8 v1, v2 offset:10
-; GFX11-NEXT:    ds_store_b8 v1, v4 offset:11
-; GFX11-NEXT:    ds_store_b8 v1, v5 offset:12
-; GFX11-NEXT:    ds_store_b8 v1, v6 offset:13
-; GFX11-NEXT:    ds_store_b8 v1, v7 offset:14
-; GFX11-NEXT:    ds_store_b8 v1, v8 offset:15
+; GFX11-NEXT:    v_mov_b16_e32 v3.l, s1
+; GFX11-NEXT:    v_mov_b16_e32 v3.h, s0
+; GFX11-NEXT:    ds_store_b8 v5, v1 offset:8
+; GFX11-NEXT:    ds_store_b8 v5, v0 offset:9
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v0 offset:10
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v1 offset:11
+; GFX11-NEXT:    ds_store_b8 v5, v2 offset:12
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v2 offset:13
+; GFX11-NEXT:    ds_store_b8 v5, v3 offset:14
+; GFX11-NEXT:    ds_store_b8_d16_hi v5, v3 offset:15
 ; GFX11-NEXT:    s_endpgm
   store <4 x i32> %x, ptr addrspace(3) %out, align 1
   ret void

llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll

@@ -207,36 +207,42 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
 ; GFX11-NEXT:    s_load_b32 s3, s[4:5], 0x0
 ; GFX11-NEXT:    s_and_b32 s5, 0xffff, s0
 ; GFX11-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX11-NEXT:    v_mov_b16_e32 v0.l, s0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT:    v_mov_b32_e32 v6, s3
 ; GFX11-NEXT:    s_lshr_b32 s0, s1, 16
 ; GFX11-NEXT:    s_and_b32 s3, 0xffff, s1
-; GFX11-NEXT:    v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT:    v_mov_b16_e32 v0.h, s1
 ; GFX11-NEXT:    s_lshr_b32 s1, s2, 16
 ; GFX11-NEXT:    s_and_b32 s6, 0xffff, s2
+; GFX11-NEXT:    v_mov_b16_e32 v1.l, s2
 ; GFX11-NEXT:    s_lshr_b32 s2, s5, 8
 ; GFX11-NEXT:    s_lshr_b32 s5, s4, 8
-; GFX11-NEXT:    v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v7, s2
-; GFX11-NEXT:    v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT:    v_mov_b16_e32 v3.l, s2
+; GFX11-NEXT:    v_mov_b16_e32 v1.h, s4
 ; GFX11-NEXT:    s_lshr_b32 s3, s3, 8
 ; GFX11-NEXT:    s_lshr_b32 s4, s0, 8
+; GFX11-NEXT:    v_mov_b16_e32 v2.l, s0
 ; GFX11-NEXT:    s_lshr_b32 s0, s6, 8
 ; GFX11-NEXT:    s_lshr_b32 s6, s1, 8
-; GFX11-NEXT:    v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v9, s3
-; GFX11-NEXT:    v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v11, s0
-; GFX11-NEXT:    v_mov_b32_e32 v12, s6
-; GFX11-NEXT:    ds_store_b8 v1, v0
-; GFX11-NEXT:    ds_store_b8 v1, v7 offset:1
-; GFX11-NEXT:    ds_store_b8 v1, v4 offset:2
-; GFX11-NEXT:    ds_store_b8 v1, v8 offset:3
-; GFX11-NEXT:    ds_store_b8 v1, v2 offset:4
-; GFX11-NEXT:    ds_store_b8 v1, v9 offset:5
-; GFX11-NEXT:    ds_store_b8 v1, v5 offset:6
-; GFX11-NEXT:    ds_store_b8 v1, v10 offset:7
-; GFX11-NEXT:    ds_store_b8 v1, v3 offset:8
-; GFX11-NEXT:    ds_store_b8 v1, v11 offset:9
-; GFX11-NEXT:    ds_store_b8 v1, v6 offset:10
-; GFX11-NEXT:    ds_store_b8 v1, v12 offset:11
+; GFX11-NEXT:    v_mov_b16_e32 v3.h, s5
+; GFX11-NEXT:    v_mov_b16_e32 v2.h, s1
+; GFX11-NEXT:    v_mov_b16_e32 v4.l, s3
+; GFX11-NEXT:    v_mov_b16_e32 v4.h, s4
+; GFX11-NEXT:    v_mov_b16_e32 v5.l, s0
+; GFX11-NEXT:    v_mov_b16_e32 v5.h, s6
+; GFX11-NEXT:    ds_store_b8 v6, v0
+; GFX11-NEXT:    ds_store_b8 v6, v3 offset:1
+; GFX11-NEXT:    ds_store_b8_d16_hi v6, v1 offset:2
+; GFX11-NEXT:    ds_store_b8_d16_hi v6, v3 offset:3
+; GFX11-NEXT:    ds_store_b8_d16_hi v6, v0 offset:4
+; GFX11-NEXT:    ds_store_b8 v6, v4 offset:5
+; GFX11-NEXT:    ds_store_b8 v6, v2 offset:6
+; GFX11-NEXT:    ds_store_b8_d16_hi v6, v4 offset:7
+; GFX11-NEXT:    ds_store_b8 v6, v1 offset:8
+; GFX11-NEXT:    ds_store_b8 v6, v5 offset:9
+; GFX11-NEXT:    ds_store_b8_d16_hi v6, v2 offset:10
+; GFX11-NEXT:    ds_store_b8_d16_hi v6, v5 offset:11
 ; GFX11-NEXT:    s_endpgm
   store <3 x i32> %x, ptr addrspace(3) %out, align 1
   ret void