diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 0ac84f4e1f02a..992f7ed99d3bb 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -784,8 +784,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Custom); setOperationAction(ISD::VECTOR_SHUFFLE, - {MVT::v4f16, MVT::v4i16, MVT::v8f16, MVT::v8i16, - MVT::v16f16, MVT::v16i16, MVT::v32f16, MVT::v32i16}, + {MVT::v4f16, MVT::v4i16, MVT::v4bf16, MVT::v8f16, + MVT::v8i16, MVT::v8bf16, MVT::v16f16, MVT::v16i16, + MVT::v16bf16, MVT::v32f16, MVT::v32i16, MVT::v32bf16}, Custom); for (MVT VT : {MVT::v4i16, MVT::v8i16, MVT::v16i16, MVT::v32i16}) @@ -7545,9 +7546,8 @@ SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op, SDLoc SL(Op); EVT ResultVT = Op.getValueType(); ShuffleVectorSDNode *SVN = cast(Op); - - EVT PackVT = ResultVT.isInteger() ? MVT::v2i16 : MVT::v2f16; - EVT EltVT = PackVT.getVectorElementType(); + MVT EltVT = ResultVT.getVectorElementType().getSimpleVT(); + MVT PackVT = MVT::getVectorVT(EltVT, 2); int SrcNumElts = Op.getOperand(0).getValueType().getVectorNumElements(); // vector_shuffle <0,1,6,7> lhs, rhs diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll index e408e83da1c29..e7ae9d831424c 100644 --- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll +++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,GX900 %s +; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx940 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,GFX940 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX11 %s @@ -31,16 +32,27 @@ define <4 x half> @shuffle_v4f16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_234u: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_234u: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_mov_b32_e32 v0, v6 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_234u: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_mov_b32_e32 v0, v4 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v6 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_234u: ; GFX10: ; %bb.0: @@ -94,13 +106,22 @@ define <4 x half> @shuffle_v4f16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_u3u1: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_u3u1: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_u3u1: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v3 +; GFX940-NEXT: v_mov_b32_e32 v1, v2 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_u3u1: ; GFX10: ; %bb.0: @@ -151,16 +172,27 @@ define <4 x half> @shuffle_v4f16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_3u6u: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_3u6u: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_3u6u: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_3u6u: ; GFX10: ; %bb.0: @@ -189,16 +221,27 @@ define <4 x half> @shuffle_v4f16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_3uu7: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_3uu7: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_3uu7: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_3uu7: ; GFX10: ; %bb.0: @@ -227,16 +270,27 @@ define <4 x half> @shuffle_v4f16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_35u5: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v4, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v4, v5, s4 -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_35u5: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GX900-NEXT: global_load_dword v4, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v4, v5, s4 +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_35u5: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dword v4, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v4, v5, s0 +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_35u5: ; GFX10: ; %bb.0: @@ -263,17 +317,29 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_357u: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v1, s4, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v4, v6, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_357u: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v1, s4, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v4, v6, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_357u: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v1, s0, v5, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v4, v6, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_357u: ; GFX10: ; %bb.0: @@ -432,13 +498,22 @@ define <4 x half> @shuffle_v4f16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_2301: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_2301: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_2301: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v3 +; GFX940-NEXT: v_mov_b32_e32 v1, v2 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2301: ; GFX10: ; %bb.0: @@ -773,13 +848,22 @@ define <4 x half> @shuffle_v4f16_6723(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_6745: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_6745: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_6745: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v3 +; GFX940-NEXT: v_mov_b32_e32 v1, v2 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6745: ; GFX10: ; %bb.0: @@ -833,16 +917,27 @@ define <4 x half> @shuffle_v4f16_6767(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_2356: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_2356: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_2356: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2356: ; GFX10: ; %bb.0: @@ -871,16 +966,27 @@ define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_5623: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_5623: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_5623: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_5623: ; GFX10: ; %bb.0: @@ -987,17 +1093,29 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_5734: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_alignbit_b32 v1, v4, v6, 16 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_5734: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_alignbit_b32 v1, v4, v6, 16 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_5734: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_alignbit_b32 v1, v4, v6, 16 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_5734: ; GFX10: ; %bb.0: @@ -1027,16 +1145,27 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4i16_2356: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4i16_2356: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4i16_2356: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i16_2356: ; GFX10: ; %bb.0: @@ -1101,15 +1230,25 @@ define <4 x i16> @shuffle_v4i16_0167(ptr addrspace(1) %arg0, ptr addrspace(1) %a } define <4 x half> @shuffle_v4f16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_0000: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4 -; GFX9-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_0000: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v0, v0, s4 +; GX900-NEXT: v_mov_b32_e32 v1, v0 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_0000: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v0, v0, s0 +; GFX940-NEXT: v_mov_b32_e32 v1, v0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0000: ; GFX10: ; %bb.0: @@ -1170,16 +1309,27 @@ define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_1100: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_mov_b32 s5, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v1, v1, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_1100: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_mov_b32 s5, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v1, v1, s4 +; GX900-NEXT: v_perm_b32 v1, v1, v1, s5 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_1100: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_mov_b32 s1, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v2, v2, s0 +; GFX940-NEXT: v_perm_b32 v1, v2, v2, s1 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_1100: ; GFX10: ; %bb.0: @@ -1205,16 +1355,27 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_6161: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v4 -; GFX9-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_6161: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0xffff +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4 +; GX900-NEXT: v_mov_b32_e32 v1, v0 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_6161: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0xffff +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_bfi_b32 v0, s0, v5, v4 +; GFX940-NEXT: v_mov_b32_e32 v1, v0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6161: ; GFX10: ; %bb.0: @@ -1243,14 +1404,23 @@ define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_2333: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_2333: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_2333: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_2333: ; GFX10: ; %bb.0: @@ -1274,14 +1444,23 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_6667: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_6667: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_6667: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_6667: ; GFX10: ; %bb.0: @@ -1436,16 +1615,27 @@ define <4 x half> @shuffle_v8f16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrspace } define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v8f16_13_14_2_3: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8f16_13_14_2_3: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8f16_13_14_2_3: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off offset:8 +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_13_14_2_3: ; GFX10: ; %bb.0: @@ -1474,14 +1664,23 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace } define <4 x half> @shuffle_v3f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v3f16_0122: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v1, v1, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v3f16_0122: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v1, v1, v1, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v3f16_0122: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v1, v1, v1, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v3f16_0122: ; GFX10: ; %bb.0: @@ -1535,20 +1734,35 @@ define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) % } define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v6f16_452367: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v6, v1 -; GFX9-NEXT: v_mov_b32_e32 v5, v0 -; GFX9-NEXT: v_mov_b32_e32 v4, v3 -; GFX9-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-NEXT: global_load_dwordx3 v[0:2], v[5:6], off -; GFX9-NEXT: global_load_dword v7, v[3:4], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v2, v7 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v6f16_452367: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v6, v1 +; GX900-NEXT: v_mov_b32_e32 v5, v0 +; GX900-NEXT: v_mov_b32_e32 v4, v3 +; GX900-NEXT: v_mov_b32_e32 v3, v2 +; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off +; GX900-NEXT: global_load_dword v7, v[3:4], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v2, v7 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v6f16_452367: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v7, v1 +; GFX940-NEXT: v_mov_b32_e32 v6, v0 +; GFX940-NEXT: v_mov_b32_e32 v5, v3 +; GFX940-NEXT: v_mov_b32_e32 v4, v2 +; GFX940-NEXT: global_load_dwordx3 v[0:2], v[6:7], off +; GFX940-NEXT: global_load_dword v3, v[4:5], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_mov_b32_e32 v0, v2 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v2, v3 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v6f16_452367: ; GFX10: ; %bb.0: @@ -1583,22 +1797,40 @@ define <6 x half> @shuffle_v6f16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) } define amdgpu_kernel void @fma_shuffle_v2f16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) { -; GFX9-LABEL: fma_shuffle_v2f16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10 -; GFX9-NEXT: v_lshlrev_b32_e32 v6, 3, v0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] -; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] -; GFX9-NEXT: global_load_dwordx2 v[4:5], v6, s[4:5] -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] -; GFX9-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] -; GFX9-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] -; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] -; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[4:5] -; GFX9-NEXT: s_endpgm +; GX900-LABEL: fma_shuffle_v2f16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GX900-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10 +; GX900-NEXT: v_lshlrev_b32_e32 v6, 3, v0 +; GX900-NEXT: s_waitcnt lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] +; GX900-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] +; GX900-NEXT: global_load_dwordx2 v[4:5], v6, s[4:5] +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] +; GX900-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] +; GX900-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] +; GX900-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] +; GX900-NEXT: global_store_dwordx2 v6, v[0:1], s[4:5] +; GX900-NEXT: s_endpgm +; +; GFX940-LABEL: fma_shuffle_v2f16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX940-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 +; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX940-NEXT: v_lshlrev_b32_e32 v6, 3, v0 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] +; GFX940-NEXT: global_load_dwordx2 v[2:3], v6, s[2:3] +; GFX940-NEXT: global_load_dwordx2 v[4:5], v6, s[6:7] +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_pk_fma_f16 v4, v0, v2, v4 op_sel_hi:[0,1,1] +; GFX940-NEXT: v_pk_fma_f16 v2, v1, v2, v5 op_sel_hi:[0,1,1] +; GFX940-NEXT: v_pk_fma_f16 v0, v0, v3, v4 op_sel:[1,0,0] +; GFX940-NEXT: v_pk_fma_f16 v1, v1, v3, v2 op_sel:[1,0,0] +; GFX940-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7] sc0 sc1 +; GFX940-NEXT: s_endpgm ; ; GFX10-LABEL: fma_shuffle_v2f16: ; GFX10: ; %bb.0: ; %entry @@ -1670,18 +1902,29 @@ entry: } define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4f16_0456: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4f16_0456: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GX900-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4f16_0456: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v6, v4, s0 +; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4f16_0456: ; GFX10: ; %bb.0: @@ -1711,19 +1954,31 @@ define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) % } define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_scalar_load_v8i32_0123: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 -; GFX9-NEXT: v_mov_b32_e32 v4, 0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s4 -; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: v_mov_b32_e32 v2, s6 -; GFX9-NEXT: v_mov_b32_e32 v3, s7 -; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] -; GFX9-NEXT: s_endpgm +; GX900-LABEL: shuffle_scalar_load_v8i32_0123: +; GX900: ; %bb.0: +; GX900-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 +; GX900-NEXT: v_mov_b32_e32 v4, 0 +; GX900-NEXT: s_waitcnt lgkmcnt(0) +; GX900-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 +; GX900-NEXT: s_waitcnt lgkmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, s4 +; GX900-NEXT: v_mov_b32_e32 v1, s5 +; GX900-NEXT: v_mov_b32_e32 v2, s6 +; GX900-NEXT: v_mov_b32_e32 v3, s7 +; GX900-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GX900-NEXT: s_endpgm +; +; GFX940-LABEL: shuffle_scalar_load_v8i32_0123: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX940-NEXT: v_mov_b32_e32 v4, 0 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: v_mov_b64_e32 v[0:1], s[4:5] +; GFX940-NEXT: v_mov_b64_e32 v[2:3], s[6:7] +; GFX940-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] sc0 sc1 +; GFX940-NEXT: s_endpgm ; ; GFX10-LABEL: shuffle_scalar_load_v8i32_0123: ; GFX10: ; %bb.0: @@ -1757,15 +2012,25 @@ define amdgpu_kernel void @shuffle_scalar_load_v8i32_0123(ptr addrspace(4) %in, } define <2 x half> @low16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: low16bits_v2f16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: low16bits_v2f16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: low16bits_v2f16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16bits_v2f16: ; GFX10: ; %bb.0: ; %entry @@ -1793,15 +2058,25 @@ entry: } define <2 x half> @hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: hi16bits_v2f16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: hi16bits_v2f16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: hi16bits_v2f16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: hi16bits_v2f16: ; GFX10: ; %bb.0: ; %entry @@ -1829,15 +2104,25 @@ entry: } define <2 x half> @low16hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: low16hi16bits_v2f16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v5 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: low16hi16bits_v2f16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0xffff +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_bfi_b32 v0, s4, v4, v5 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: low16hi16bits_v2f16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0xffff +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_bfi_b32 v0, s0, v4, v5 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16hi16bits_v2f16: ; GFX10: ; %bb.0: ; %entry @@ -1900,15 +2185,25 @@ entry: } define <2 x i16> @i16_low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: i16_low16bits: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: i16_low16bits: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: i16_low16bits: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_low16bits: ; GFX10: ; %bb.0: ; %entry @@ -1936,15 +2231,25 @@ entry: } define <2 x i16> @i16_low16hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: i16_low16hi16bits: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v5 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: i16_low16hi16bits: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0xffff +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_bfi_b32 v0, s4, v4, v5 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: i16_low16hi16bits: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0xffff +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_bfi_b32 v0, s0, v4, v5 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_low16hi16bits: ; GFX10: ; %bb.0: ; %entry @@ -2007,15 +2312,25 @@ entry: } define <2 x i16> @i16_hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: i16_hi16bits: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: i16_hi16bits: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: i16_hi16bits: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: i16_hi16bits: ; GFX10: ; %bb.0: ; %entry @@ -2107,15 +2422,25 @@ entry: } define void @shuffle_v8f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v8f16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8f16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8f16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8f16_concat: ; GFX10: ; %bb.0: @@ -2142,17 +2467,29 @@ define void @shuffle_v8f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1 } define void @shuffle_v16f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v16f16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v16f16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v16f16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16f16_concat: ; GFX10: ; %bb.0: @@ -2183,23 +2520,41 @@ define void @shuffle_v16f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg } define void @shuffle_v32f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v32f16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 -; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off -; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v32f16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v32f16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32f16_concat: ; GFX10: ; %bb.0: @@ -2246,15 +2601,25 @@ define void @shuffle_v32f16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg } define void @shuffle_v8i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v8i16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8i16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8i16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8i16_concat: ; GFX10: ; %bb.0: @@ -2281,17 +2646,29 @@ define void @shuffle_v8i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1 } define void @shuffle_v16i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v16i16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v16i16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v16i16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16i16_concat: ; GFX10: ; %bb.0: @@ -2322,23 +2699,41 @@ define void @shuffle_v16i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg } define void @shuffle_v32i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v32i16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 -; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off -; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v32i16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v32i16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32i16_concat: ; GFX10: ; %bb.0: @@ -2385,16 +2780,28 @@ define void @shuffle_v32i16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg } define void @shuffle_v4i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v4i8_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_ushort v0, v[0:1], off -; GFX9-NEXT: s_nop 0 -; GFX9-NEXT: global_load_short_d16_hi v0, v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dword v[4:5], v0, off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4i8_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_ushort v0, v[0:1], off +; GX900-NEXT: s_nop 0 +; GX900-NEXT: global_load_short_d16_hi v0, v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dword v[4:5], v0, off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4i8_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_ushort v6, v[0:1], off +; GFX940-NEXT: global_load_ushort v7, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v7, v6, s0 +; GFX940-NEXT: global_store_dword v[4:5], v0, off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i8_concat: ; GFX10: ; %bb.0: @@ -2421,15 +2828,25 @@ define void @shuffle_v4i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, } define void @shuffle_v8i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v8i8_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v6, v[0:1], off -; GFX9-NEXT: global_load_dword v7, v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dwordx2 v[4:5], v[6:7], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8i8_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v6, v[0:1], off +; GX900-NEXT: global_load_dword v7, v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dwordx2 v[4:5], v[6:7], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8i8_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v6, v[0:1], off +; GFX940-NEXT: global_load_dword v7, v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: global_store_dwordx2 v[4:5], v[6:7], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8i8_concat: ; GFX10: ; %bb.0: @@ -2456,15 +2873,25 @@ define void @shuffle_v8i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, } define void @shuffle_v16i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v16i8_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v16i8_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v16i8_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16i8_concat: ; GFX10: ; %bb.0: @@ -2491,17 +2918,29 @@ define void @shuffle_v16i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1 } define void @shuffle_v32i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v32i8_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v32i8_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v32i8_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32i8_concat: ; GFX10: ; %bb.0: @@ -2532,15 +2971,25 @@ define void @shuffle_v32i8_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1 } define void @shuffle_v4i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v4i32_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4i32_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4i32_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4i32_concat: ; GFX10: ; %bb.0: @@ -2567,17 +3016,29 @@ define void @shuffle_v4i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1 } define void @shuffle_v8i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v8i32_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8i32_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8i32_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8i32_concat: ; GFX10: ; %bb.0: @@ -2608,23 +3069,41 @@ define void @shuffle_v8i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1 } define void @shuffle_v16i32_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v16i32_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 -; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off -; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v16i32_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v16i32_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16i32_concat: ; GFX10: ; %bb.0: @@ -2698,16 +3177,27 @@ define <4 x bfloat> @shuffle_v4bf16_23uu(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_234u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_234u: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v6 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_234u: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_mov_b32_e32 v0, v6 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_234u: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_mov_b32_e32 v0, v4 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v6 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_234u: ; GFX10: ; %bb.0: @@ -2761,13 +3251,22 @@ define <4 x bfloat> @shuffle_v4bf16_u1u3(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_u3u1(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_u3u1: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_u3u1: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_u3u1: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v3 +; GFX940-NEXT: v_mov_b32_e32 v1, v2 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_u3u1: ; GFX10: ; %bb.0: @@ -2818,16 +3317,27 @@ define <4 x bfloat> @shuffle_v4bf16_u3uu(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_3u6u: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_3u6u: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_3u6u: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_3u6u: ; GFX10: ; %bb.0: @@ -2856,16 +3366,27 @@ define <4 x bfloat> @shuffle_v4bf16_3u6u(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_3uu7: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, s4, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_3uu7: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GX900-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, s4, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_3uu7: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, s0, v5, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_3uu7: ; GFX10: ; %bb.0: @@ -2894,16 +3415,27 @@ define <4 x bfloat> @shuffle_v4bf16_3uu7(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_35u5: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 -; GFX9-NEXT: global_load_dword v4, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v4, v5, s4 -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_35u5: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GX900-NEXT: global_load_dword v4, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v4, v5, s4 +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_35u5: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v5, v[0:1], off offset:4 +; GFX940-NEXT: global_load_dword v4, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v4, v5, s0 +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_35u5: ; GFX10: ; %bb.0: @@ -2930,17 +3462,29 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_357u: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v1, s4, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v4, v6, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_357u: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v1, s4, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v4, v6, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_357u: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v1, s0, v5, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v4, v6, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_357u: ; GFX10: ; %bb.0: @@ -2973,29 +3517,24 @@ define <4 x bfloat> @shuffle_v4bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3035,39 +3574,31 @@ define <4 x bfloat> @shuffle_v4bf16_0145(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0145: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GFX9-NEXT: global_load_dword v4, v[0:1], off +; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0145: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GFX10-NEXT: global_load_dword v4, v[0:1], off +; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0145: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off -; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off -; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 +; GFX11-NEXT: global_load_b32 v0, v[0:1], off +; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 @@ -3079,33 +3610,30 @@ define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_0167: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off -; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v4, v[0:1], off +; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v5 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0167: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off -; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX10-NEXT: global_load_dword v4, v[0:1], off +; GFX10-NEXT: global_load_dword v5, v[2:3], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v4 +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_0167: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: global_load_b32 v1, v[2:3], off offset:4 -; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3115,22 +3643,28 @@ define <4 x bfloat> @shuffle_v4bf16_0167(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_2301: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_2301: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_2301: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v3 +; GFX940-NEXT: v_mov_b32_e32 v1, v2 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2301: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; @@ -3139,7 +3673,6 @@ define <4 x bfloat> @shuffle_v4bf16_2301(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3182,33 +3715,30 @@ define <4 x bfloat> @shuffle_v4bf16_2345(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_2345: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 -; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2345: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off ; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 -; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_2345: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off ; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4 -; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v2, v2 +; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3257,39 +3787,33 @@ define <4 x bfloat> @shuffle_v4bf16_4501(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_4501: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GFX9-NEXT: global_load_dword v4, v[2:3], off +; GFX9-NEXT: global_load_dword v5, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4501: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off -; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GFX10-NEXT: global_load_dword v4, v[2:3], off +; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4501: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off -; GFX11-NEXT: global_load_b64 v[3:4], v[0:1], off +; GFX11-NEXT: global_load_b32 v2, v[2:3], off +; GFX11-NEXT: global_load_b32 v1, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v2, v2 +; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v3, v3 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <4 x bfloat>, ptr addrspace(1) %arg1 @@ -3301,33 +3825,32 @@ define <4 x bfloat> @shuffle_v4bf16_4523(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_4523: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v4, v[2:3], off +; GFX9-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v5 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4523: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX10-NEXT: global_load_dword v4, v[2:3], off +; GFX10-NEXT: global_load_dword v5, v[0:1], off offset:4 ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v5, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v1, v4 +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4523: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[2:3], v[2:3], off +; GFX11-NEXT: global_load_b32 v2, v[2:3], off ; GFX11-NEXT: global_load_b32 v1, v[0:1], off offset:4 ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v2, v2 +; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3340,29 +3863,24 @@ define <4 x bfloat> @shuffle_v4bf16_4545(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_4545: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v0, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_4545: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[0:1], v[2:3], off +; GFX10-NEXT: global_load_dword v0, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_4545: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[0:1], v[2:3], off +; GFX11-NEXT: global_load_b32 v0, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3402,33 +3920,32 @@ define <4 x bfloat> @shuffle_v4bf16_6701(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v4bf16_6701: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[0:1], off ; GFX9-NEXT: global_load_dword v4, v[2:3], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v5, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 -; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mov_b32_e32 v0, v4 +; GFX9-NEXT: s_waitcnt vmcnt(0) +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6701: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[0:1], off ; GFX10-NEXT: global_load_dword v4, v[2:3], off offset:4 +; GFX10-NEXT: global_load_dword v5, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 -; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_mov_b32_e32 v0, v4 +; GFX10-NEXT: s_waitcnt vmcnt(0) +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v4bf16_6701: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off -; GFX11-NEXT: global_load_b32 v0, v[2:3], off offset:4 +; GFX11-NEXT: global_load_b32 v2, v[2:3], off offset:4 +; GFX11-NEXT: global_load_b32 v1, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v4, v4 +; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_waitcnt vmcnt(0) ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3476,22 +3993,28 @@ define <4 x bfloat> @shuffle_v4bf16_6723(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_6745: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_6745: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_6745: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v3 +; GFX940-NEXT: v_mov_b32_e32 v1, v2 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6745: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[1:2], v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX10-NEXT: v_mov_b32_e32 v0, v2 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; @@ -3500,7 +4023,6 @@ define <4 x bfloat> @shuffle_v4bf16_6745(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: v_mov_b32_e32 v0, v2 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <4 x bfloat>, ptr addrspace(1) %arg0 @@ -3540,16 +4062,27 @@ define <4 x bfloat> @shuffle_v4bf16_6767(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_2356: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_2356: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v0, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_2356: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v0, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2356: ; GFX10: ; %bb.0: @@ -3578,16 +4111,27 @@ define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_5623: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_5623: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_5623: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_5623: ; GFX10: ; %bb.0: @@ -3694,17 +4238,29 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_5734: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[2:3], off -; GFX9-NEXT: global_load_dword v6, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_alignbit_b32 v1, v4, v6, 16 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_5734: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GX900-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_alignbit_b32 v1, v4, v6, 16 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_5734: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[4:5], v[2:3], off +; GFX940-NEXT: global_load_dword v6, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_alignbit_b32 v1, v4, v6, 16 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_5734: ; GFX10: ; %bb.0: @@ -3734,15 +4290,25 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_0000(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_0000: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4 -; GFX9-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_0000: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v0, v0, s4 +; GX900-NEXT: v_mov_b32_e32 v1, v0 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_0000: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v0, v0, s0 +; GFX940-NEXT: v_mov_b32_e32 v1, v0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0000: ; GFX10: ; %bb.0: @@ -3803,16 +4369,27 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_1100: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_mov_b32 s5, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v1, v1, s4 -; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_1100: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_mov_b32 s5, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v1, v1, s4 +; GX900-NEXT: v_perm_b32 v1, v1, v1, s5 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_1100: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[2:3], v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_mov_b32 s1, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v2, v2, s0 +; GFX940-NEXT: v_perm_b32 v1, v2, v2, s1 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_1100: ; GFX10: ; %bb.0: @@ -3838,16 +4415,27 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_6161: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v4 -; GFX9-NEXT: v_mov_b32_e32 v1, v0 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_6161: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0xffff +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4 +; GX900-NEXT: v_mov_b32_e32 v1, v0 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_6161: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0xffff +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_bfi_b32 v0, s0, v5, v4 +; GFX940-NEXT: v_mov_b32_e32 v1, v0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6161: ; GFX10: ; %bb.0: @@ -3876,14 +4464,23 @@ define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_2333: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_2333: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_2333: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_2333: ; GFX10: ; %bb.0: @@ -3907,14 +4504,23 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1 } define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_6667: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v0, v[0:1], off offset:4 -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v1, v0, v0, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_6667: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v1, v0, v0, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_6667: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v0, v[0:1], off offset:4 +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v1, v0, v0, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_6667: ; GFX10: ; %bb.0: @@ -3941,29 +4547,24 @@ define <4 x bfloat> @shuffle_v8bf16_0101(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v8bf16_0101: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff +; GFX9-NEXT: global_load_dword v0, v[0:1], off ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 ; GFX9-NEXT: v_mov_b32_e32 v1, v0 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_0101: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX10-NEXT: global_load_dword v0, v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_mov_b32_e32 v1, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_0101: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off +; GFX11-NEXT: global_load_b32 v0, v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) ; GFX11-NEXT: v_mov_b32_e32 v1, v0 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 @@ -4003,39 +4604,31 @@ define <4 x bfloat> @shuffle_v8bf16_4589(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX9-LABEL: shuffle_v8bf16_4589: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:8 -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:8 +; GFX9-NEXT: global_load_dword v5, v[2:3], off ; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_bfi_b32 v0, s4, v4, v4 +; GFX9-NEXT: v_mov_b32_e32 v0, v4 ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v1, s4, v5, v5 +; GFX9-NEXT: v_mov_b32_e32 v1, v5 ; GFX9-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_4589: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:8 -; GFX10-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX10-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX10-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:8 +; GFX10-NEXT: global_load_dword v5, v[2:3], off ; GFX10-NEXT: s_waitcnt vmcnt(1) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v4 +; GFX10-NEXT: v_mov_b32_e32 v0, v4 ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v1, 0xffff, v5, v5 +; GFX10-NEXT: v_mov_b32_e32 v1, v5 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; ; GFX11-LABEL: shuffle_v8bf16_4589: ; GFX11: ; %bb.0: ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off offset:8 -; GFX11-NEXT: global_load_b64 v[1:2], v[2:3], off -; GFX11-NEXT: s_waitcnt vmcnt(1) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 +; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:8 +; GFX11-NEXT: global_load_b32 v1, v[2:3], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v1, 0xffff, v1, v1 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <8 x bfloat>, ptr addrspace(1) %arg0 %val1 = load <8 x bfloat>, ptr addrspace(1) %arg1 @@ -4082,16 +4675,27 @@ define <4 x bfloat> @shuffle_v8bf16_10_11_2_3(ptr addrspace(1) %arg0, ptr addrsp } define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v8bf16_13_14_2_3: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 -; GFX9-NEXT: global_load_dword v4, v[0:1], off offset:4 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_alignbit_b32 v0, v6, v5, 16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8bf16_13_14_2_3: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off offset:8 +; GX900-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_alignbit_b32 v0, v6, v5, 16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v1, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8bf16_13_14_2_3: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off offset:8 +; GFX940-NEXT: global_load_dword v4, v[0:1], off offset:4 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_alignbit_b32 v0, v7, v6, 16 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v1, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_13_14_2_3: ; GFX10: ; %bb.0: @@ -4120,23 +4724,29 @@ define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrsp } define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v3bf16_0122: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_mov_b32 s5, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v0 -; GFX9-NEXT: v_perm_b32 v1, v1, v1, s5 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v3bf16_0122: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v1, v1, v1, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v3bf16_0122: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[0:1], v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v1, v1, v1, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v3bf16_0122: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX10-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX10-NEXT: s_setpc_b64 s[30:31] ; @@ -4145,7 +4755,6 @@ define <4 x bfloat> @shuffle_v3bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1 ; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off ; GFX11-NEXT: s_waitcnt vmcnt(0) -; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v0 ; GFX11-NEXT: v_perm_b32 v1, v1, v1, 0x5040100 ; GFX11-NEXT: s_setpc_b64 s[30:31] %val0 = load <3 x bfloat>, ptr addrspace(1) %arg0 @@ -4185,20 +4794,35 @@ define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1 } define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v6bf16_452367: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v6, v1 -; GFX9-NEXT: v_mov_b32_e32 v5, v0 -; GFX9-NEXT: v_mov_b32_e32 v4, v3 -; GFX9-NEXT: v_mov_b32_e32 v3, v2 -; GFX9-NEXT: global_load_dwordx3 v[0:2], v[5:6], off -; GFX9-NEXT: global_load_dword v7, v[3:4], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_mov_b32_e32 v0, v2 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v2, v7 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v6bf16_452367: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v6, v1 +; GX900-NEXT: v_mov_b32_e32 v5, v0 +; GX900-NEXT: v_mov_b32_e32 v4, v3 +; GX900-NEXT: v_mov_b32_e32 v3, v2 +; GX900-NEXT: global_load_dwordx3 v[0:2], v[5:6], off +; GX900-NEXT: global_load_dword v7, v[3:4], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_mov_b32_e32 v0, v2 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_mov_b32_e32 v2, v7 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v6bf16_452367: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v7, v1 +; GFX940-NEXT: v_mov_b32_e32 v6, v0 +; GFX940-NEXT: v_mov_b32_e32 v5, v3 +; GFX940-NEXT: v_mov_b32_e32 v4, v2 +; GFX940-NEXT: global_load_dwordx3 v[0:2], v[6:7], off +; GFX940-NEXT: global_load_dword v3, v[4:5], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_mov_b32_e32 v0, v2 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_mov_b32_e32 v2, v3 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v6bf16_452367: ; GFX10: ; %bb.0: @@ -4233,88 +4857,173 @@ define <6 x bfloat> @shuffle_v6bf16_452367(ptr addrspace(1) %arg0, ptr addrspace } define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonly %A, ptr addrspace(1) nocapture readonly %B, ptr addrspace(1) nocapture %C) { -; GFX9-LABEL: fma_shuffle_v2bf16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0 -; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0 -; GFX9-NEXT: s_movk_i32 s2, 0x7fff -; GFX9-NEXT: s_mov_b32 s3, 0x7060302 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1] -; GFX9-NEXT: global_load_dwordx2 v[3:4], v0, s[4:5] -; GFX9-NEXT: global_load_dwordx2 v[5:6], v0, s[6:7] -; GFX9-NEXT: s_waitcnt vmcnt(2) -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: v_lshlrev_b32_e32 v8, 16, v3 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_and_b32_e32 v9, 0xffff0000, v5 -; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; GFX9-NEXT: v_and_b32_e32 v11, 0xffff0000, v2 -; GFX9-NEXT: v_lshlrev_b32_e32 v12, 16, v4 -; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX9-NEXT: v_fma_f32 v7, v8, v9, v7 -; GFX9-NEXT: v_fma_f32 v1, v8, v5, v1 -; GFX9-NEXT: v_fma_f32 v2, v12, v5, v2 -; GFX9-NEXT: v_bfe_u32 v5, v7, 16, 1 -; GFX9-NEXT: v_fma_f32 v8, v12, v9, v11 -; GFX9-NEXT: v_or_b32_e32 v9, 0x400000, v7 -; GFX9-NEXT: v_bfe_u32 v11, v1, 16, 1 -; GFX9-NEXT: v_add3_u32 v5, v5, v7, s2 -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 -; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v1 -; GFX9-NEXT: v_bfe_u32 v13, v8, 16, 1 -; GFX9-NEXT: v_add3_u32 v11, v11, v1, s2 -; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; GFX9-NEXT: v_or_b32_e32 v14, 0x400000, v8 -; GFX9-NEXT: v_bfe_u32 v15, v2, 16, 1 -; GFX9-NEXT: v_add3_u32 v13, v13, v8, s2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v8, v8 -; GFX9-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 -; GFX9-NEXT: v_lshlrev_b32_e32 v10, 16, v6 -; GFX9-NEXT: v_or_b32_e32 v16, 0x400000, v2 -; GFX9-NEXT: v_add3_u32 v15, v15, v2, s2 -; GFX9-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 -; GFX9-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc -; GFX9-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 -; GFX9-NEXT: v_fma_f32 v1, v3, v10, v1 -; GFX9-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 -; GFX9-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 -; GFX9-NEXT: v_fma_f32 v3, v3, v6, v5 -; GFX9-NEXT: v_bfe_u32 v5, v1, 16, 1 -; GFX9-NEXT: v_fma_f32 v2, v4, v10, v2 -; GFX9-NEXT: v_fma_f32 v4, v4, v6, v7 -; GFX9-NEXT: v_or_b32_e32 v6, 0x400000, v1 -; GFX9-NEXT: v_bfe_u32 v7, v3, 16, 1 -; GFX9-NEXT: v_add3_u32 v5, v5, v1, s2 -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 -; GFX9-NEXT: v_or_b32_e32 v8, 0x400000, v3 -; GFX9-NEXT: v_bfe_u32 v9, v2, 16, 1 -; GFX9-NEXT: v_add3_u32 v7, v7, v3, s2 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 -; GFX9-NEXT: v_or_b32_e32 v10, 0x400000, v2 -; GFX9-NEXT: v_bfe_u32 v11, v4, 16, 1 -; GFX9-NEXT: v_add3_u32 v9, v9, v2, s2 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 -; GFX9-NEXT: v_or_b32_e32 v12, 0x400000, v4 -; GFX9-NEXT: v_add3_u32 v11, v11, v4, s2 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc -; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc -; GFX9-NEXT: v_perm_b32 v2, v4, v2, s3 -; GFX9-NEXT: v_perm_b32 v1, v3, v1, s3 -; GFX9-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1] -; GFX9-NEXT: s_endpgm +; GX900-LABEL: fma_shuffle_v2bf16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10 +; GX900-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0 +; GX900-NEXT: v_lshlrev_b32_e32 v0, 3, v0 +; GX900-NEXT: s_movk_i32 s2, 0x7fff +; GX900-NEXT: s_mov_b32 s3, 0x7060302 +; GX900-NEXT: s_waitcnt lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[1:2], v0, s[0:1] +; GX900-NEXT: global_load_dwordx2 v[3:4], v0, s[4:5] +; GX900-NEXT: global_load_dwordx2 v[5:6], v0, s[6:7] +; GX900-NEXT: s_waitcnt vmcnt(2) +; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v1 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: v_lshlrev_b32_e32 v8, 16, v3 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_and_b32_e32 v9, 0xffff0000, v5 +; GX900-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GX900-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GX900-NEXT: v_and_b32_e32 v11, 0xffff0000, v2 +; GX900-NEXT: v_lshlrev_b32_e32 v12, 16, v4 +; GX900-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GX900-NEXT: v_fma_f32 v7, v8, v9, v7 +; GX900-NEXT: v_fma_f32 v1, v8, v5, v1 +; GX900-NEXT: v_fma_f32 v2, v12, v5, v2 +; GX900-NEXT: v_bfe_u32 v5, v7, 16, 1 +; GX900-NEXT: v_fma_f32 v8, v12, v9, v11 +; GX900-NEXT: v_or_b32_e32 v9, 0x400000, v7 +; GX900-NEXT: v_bfe_u32 v11, v1, 16, 1 +; GX900-NEXT: v_add3_u32 v5, v5, v7, s2 +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 +; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v1 +; GX900-NEXT: v_bfe_u32 v13, v8, 16, 1 +; GX900-NEXT: v_add3_u32 v11, v11, v1, s2 +; GX900-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 +; GX900-NEXT: v_or_b32_e32 v14, 0x400000, v8 +; GX900-NEXT: v_bfe_u32 v15, v2, 16, 1 +; GX900-NEXT: v_add3_u32 v13, v13, v8, s2 +; GX900-NEXT: v_cndmask_b32_e32 v1, v11, v12, vcc +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v8, v8 +; GX900-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GX900-NEXT: v_lshlrev_b32_e32 v10, 16, v6 +; GX900-NEXT: v_or_b32_e32 v16, 0x400000, v2 +; GX900-NEXT: v_add3_u32 v15, v15, v2, s2 +; GX900-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 +; GX900-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GX900-NEXT: v_and_b32_e32 v6, 0xffff0000, v6 +; GX900-NEXT: v_cndmask_b32_e32 v2, v15, v16, vcc +; GX900-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GX900-NEXT: v_fma_f32 v1, v3, v10, v1 +; GX900-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GX900-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GX900-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GX900-NEXT: v_fma_f32 v3, v3, v6, v5 +; GX900-NEXT: v_bfe_u32 v5, v1, 16, 1 +; GX900-NEXT: v_fma_f32 v2, v4, v10, v2 +; GX900-NEXT: v_fma_f32 v4, v4, v6, v7 +; GX900-NEXT: v_or_b32_e32 v6, 0x400000, v1 +; GX900-NEXT: v_bfe_u32 v7, v3, 16, 1 +; GX900-NEXT: v_add3_u32 v5, v5, v1, s2 +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 +; GX900-NEXT: v_or_b32_e32 v8, 0x400000, v3 +; GX900-NEXT: v_bfe_u32 v9, v2, 16, 1 +; GX900-NEXT: v_add3_u32 v7, v7, v3, s2 +; GX900-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v3, v3 +; GX900-NEXT: v_or_b32_e32 v10, 0x400000, v2 +; GX900-NEXT: v_bfe_u32 v11, v4, 16, 1 +; GX900-NEXT: v_add3_u32 v9, v9, v2, s2 +; GX900-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v2, v2 +; GX900-NEXT: v_or_b32_e32 v12, 0x400000, v4 +; GX900-NEXT: v_add3_u32 v11, v11, v4, s2 +; GX900-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc +; GX900-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 +; GX900-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc +; GX900-NEXT: v_perm_b32 v2, v4, v2, s3 +; GX900-NEXT: v_perm_b32 v1, v3, v1, s3 +; GX900-NEXT: global_store_dwordx2 v0, v[1:2], s[0:1] +; GX900-NEXT: s_endpgm +; +; GFX940-LABEL: fma_shuffle_v2bf16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 +; GFX940-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0 +; GFX940-NEXT: v_and_b32_e32 v0, 0x3ff, v0 +; GFX940-NEXT: v_lshlrev_b32_e32 v6, 3, v0 +; GFX940-NEXT: s_movk_i32 s2, 0x7fff +; GFX940-NEXT: s_mov_b32 s3, 0x7060302 +; GFX940-NEXT: s_waitcnt lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[0:1], v6, s[0:1] +; GFX940-NEXT: global_load_dwordx2 v[2:3], v6, s[8:9] +; GFX940-NEXT: global_load_dwordx2 v[4:5], v6, s[10:11] +; GFX940-NEXT: s_waitcnt vmcnt(2) +; GFX940-NEXT: v_and_b32_e32 v7, 0xffff0000, v0 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: v_lshlrev_b32_e32 v8, 16, v2 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_and_b32_e32 v9, 0xffff0000, v4 +; GFX940-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; GFX940-NEXT: v_lshlrev_b32_e32 v4, 16, v4 +; GFX940-NEXT: v_and_b32_e32 v11, 0xffff0000, v1 +; GFX940-NEXT: v_lshlrev_b32_e32 v12, 16, v3 +; GFX940-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX940-NEXT: v_fmac_f32_e32 v7, v8, v9 +; GFX940-NEXT: v_fmac_f32_e32 v0, v8, v4 +; GFX940-NEXT: v_fmac_f32_e32 v1, v12, v4 +; GFX940-NEXT: v_bfe_u32 v4, v7, 16, 1 +; GFX940-NEXT: v_fmac_f32_e32 v11, v12, v9 +; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v7 +; GFX940-NEXT: v_bfe_u32 v9, v0, 16, 1 +; GFX940-NEXT: v_add3_u32 v4, v4, v7, s2 +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 +; GFX940-NEXT: v_or_b32_e32 v12, 0x400000, v0 +; GFX940-NEXT: v_bfe_u32 v13, v11, 16, 1 +; GFX940-NEXT: v_add3_u32 v9, v9, v0, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 +; GFX940-NEXT: v_or_b32_e32 v14, 0x400000, v11 +; GFX940-NEXT: v_bfe_u32 v15, v1, 16, 1 +; GFX940-NEXT: v_add3_u32 v13, v13, v11, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v11, v11 +; GFX940-NEXT: v_and_b32_e32 v2, 0xffff0000, v2 +; GFX940-NEXT: v_lshlrev_b32_e32 v10, 16, v5 +; GFX940-NEXT: v_or_b32_e32 v16, 0x400000, v1 +; GFX940-NEXT: v_add3_u32 v15, v15, v1, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 +; GFX940-NEXT: v_and_b32_e32 v0, 0xffff0000, v0 +; GFX940-NEXT: v_and_b32_e32 v5, 0xffff0000, v5 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc +; GFX940-NEXT: v_and_b32_e32 v4, 0xffff0000, v4 +; GFX940-NEXT: v_fmac_f32_e32 v0, v2, v10 +; GFX940-NEXT: v_and_b32_e32 v3, 0xffff0000, v3 +; GFX940-NEXT: v_and_b32_e32 v1, 0xffff0000, v1 +; GFX940-NEXT: v_and_b32_e32 v7, 0xffff0000, v7 +; GFX940-NEXT: v_fmac_f32_e32 v4, v2, v5 +; GFX940-NEXT: v_bfe_u32 v2, v0, 16, 1 +; GFX940-NEXT: v_fmac_f32_e32 v1, v3, v10 +; GFX940-NEXT: v_fmac_f32_e32 v7, v3, v5 +; GFX940-NEXT: v_or_b32_e32 v3, 0x400000, v0 +; GFX940-NEXT: v_bfe_u32 v5, v4, 16, 1 +; GFX940-NEXT: v_add3_u32 v2, v2, v0, s2 +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v0, v0 +; GFX940-NEXT: v_or_b32_e32 v8, 0x400000, v4 +; GFX940-NEXT: v_bfe_u32 v9, v1, 16, 1 +; GFX940-NEXT: v_add3_u32 v5, v5, v4, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v4, v4 +; GFX940-NEXT: v_or_b32_e32 v10, 0x400000, v1 +; GFX940-NEXT: v_bfe_u32 v11, v7, 16, 1 +; GFX940-NEXT: v_add3_u32 v9, v9, v1, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v1, v1 +; GFX940-NEXT: v_or_b32_e32 v12, 0x400000, v7 +; GFX940-NEXT: v_add3_u32 v11, v11, v7, s2 +; GFX940-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc +; GFX940-NEXT: v_cmp_u_f32_e32 vcc, v7, v7 +; GFX940-NEXT: v_perm_b32 v0, v2, v0, s3 +; GFX940-NEXT: s_nop 0 +; GFX940-NEXT: v_cndmask_b32_e32 v3, v11, v12, vcc +; GFX940-NEXT: v_perm_b32 v1, v3, v1, s3 +; GFX940-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1] sc0 sc1 +; GFX940-NEXT: s_endpgm ; ; GFX10-LABEL: fma_shuffle_v2bf16: ; GFX10: ; %bb.0: ; %entry @@ -4523,18 +5232,29 @@ entry: } define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1) { -; GFX9-LABEL: shuffle_v4bf16_0456: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[4:5], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[5:6], v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: ; kill: killed $vgpr0 killed $vgpr1 -; GFX9-NEXT: ; kill: killed $vgpr2 killed $vgpr3 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: v_alignbit_b32 v1, v6, v5, 16 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v4bf16_0456: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[5:6], v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: ; kill: killed $vgpr0 killed $vgpr1 +; GX900-NEXT: ; kill: killed $vgpr2 killed $vgpr3 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: v_alignbit_b32 v1, v6, v5, 16 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v4bf16_0456: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[4:5], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v6, v4, s0 +; GFX940-NEXT: v_alignbit_b32 v1, v7, v6, 16 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v4bf16_0456: ; GFX10: ; %bb.0: @@ -4564,15 +5284,25 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1 } define <2 x bfloat> @low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: low16bits: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x5040100 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: low16bits: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x5040100 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: low16bits: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x5040100 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16bits: ; GFX10: ; %bb.0: ; %entry @@ -4600,15 +5330,25 @@ entry: } define <2 x bfloat> @hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: hi16bits_v2bf16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[0:1], off -; GFX9-NEXT: global_load_dword v5, v[2:3], off -; GFX9-NEXT: s_mov_b32 s4, 0x7060302 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_perm_b32 v0, v5, v4, s4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: hi16bits_v2bf16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[0:1], off +; GX900-NEXT: global_load_dword v5, v[2:3], off +; GX900-NEXT: s_mov_b32 s4, 0x7060302 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_perm_b32 v0, v5, v4, s4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: hi16bits_v2bf16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[0:1], off +; GFX940-NEXT: global_load_dword v5, v[2:3], off +; GFX940-NEXT: s_mov_b32 s0, 0x7060302 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_perm_b32 v0, v5, v4, s0 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: hi16bits_v2bf16: ; GFX10: ; %bb.0: ; %entry @@ -4636,15 +5376,25 @@ entry: } define <2 x bfloat> @low16hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) { -; GFX9-LABEL: low16hi16bits_v2bf16: -; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dword v4, v[2:3], off -; GFX9-NEXT: global_load_dword v5, v[0:1], off -; GFX9-NEXT: s_mov_b32 s4, 0xffff -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_bfi_b32 v0, s4, v5, v4 -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: low16hi16bits_v2bf16: +; GX900: ; %bb.0: ; %entry +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dword v4, v[2:3], off +; GX900-NEXT: global_load_dword v5, v[0:1], off +; GX900-NEXT: s_mov_b32 s4, 0xffff +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: v_bfi_b32 v0, s4, v5, v4 +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: low16hi16bits_v2bf16: +; GFX940: ; %bb.0: ; %entry +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dword v4, v[2:3], off +; GFX940-NEXT: global_load_dword v5, v[0:1], off +; GFX940-NEXT: s_mov_b32 s0, 0xffff +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: v_bfi_b32 v0, s0, v5, v4 +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: low16hi16bits_v2bf16: ; GFX10: ; %bb.0: ; %entry @@ -4739,15 +5489,25 @@ entry: } define void @shuffle_v8bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v8bf16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[6:7], v[0:1], off -; GFX9-NEXT: global_load_dwordx2 v[8:9], v[2:3], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v8bf16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GX900-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v8bf16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx2 v[6:7], v[0:1], off +; GFX940-NEXT: global_load_dwordx2 v[8:9], v[2:3], off +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v8bf16_concat: ; GFX10: ; %bb.0: @@ -4774,17 +5534,29 @@ define void @shuffle_v8bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg } define void @shuffle_v16bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v16bf16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[0:1], off -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(1) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v16bf16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(1) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v16bf16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[0:1], off +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(1) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v16bf16_concat: ; GFX10: ; %bb.0: @@ -4815,23 +5587,41 @@ define void @shuffle_v16bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %ar } define void @shuffle_v32bf16_concat(ptr addrspace(1) %arg0, ptr addrspace(1) %arg1, ptr addrspace(1) %out) { -; GFX9-LABEL: shuffle_v32bf16_concat: -; GFX9: ; %bb.0: -; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx4 v[6:9], v[2:3], off -; GFX9-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 -; GFX9-NEXT: global_load_dwordx4 v[14:17], v[0:1], off -; GFX9-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[14:17], off -; GFX9-NEXT: s_waitcnt vmcnt(3) -; GFX9-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 -; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: s_setpc_b64 s[30:31] +; GX900-LABEL: shuffle_v32bf16_concat: +; GX900: ; %bb.0: +; GX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GX900-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GX900-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GX900-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GX900-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[14:17], off +; GX900-NEXT: s_waitcnt vmcnt(3) +; GX900-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 +; GX900-NEXT: s_waitcnt vmcnt(0) +; GX900-NEXT: s_setpc_b64 s[30:31] +; +; GFX940-LABEL: shuffle_v32bf16_concat: +; GFX940: ; %bb.0: +; GFX940-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX940-NEXT: global_load_dwordx4 v[6:9], v[2:3], off +; GFX940-NEXT: global_load_dwordx4 v[10:13], v[2:3], off offset:16 +; GFX940-NEXT: global_load_dwordx4 v[14:17], v[0:1], off +; GFX940-NEXT: global_load_dwordx4 v[18:21], v[0:1], off offset:16 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[6:9], off offset:32 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[10:13], off offset:48 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[14:17], off sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(3) +; GFX940-NEXT: global_store_dwordx4 v[4:5], v[18:21], off offset:16 sc0 sc1 +; GFX940-NEXT: s_waitcnt vmcnt(0) +; GFX940-NEXT: s_setpc_b64 s[30:31] ; ; GFX10-LABEL: shuffle_v32bf16_concat: ; GFX10: ; %bb.0: