From 111f2fd9cfa6f120b668f6f61cbd68464bfd0137 Mon Sep 17 00:00:00 2001 From: Igor Kirillov Date: Mon, 16 Dec 2024 14:41:51 +0000 Subject: [PATCH 1/2] [AArch64] Extend vecreduce to udot/sdot transformation to support usdot --- .../Target/AArch64/AArch64ISelLowering.cpp | 34 +- llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 5442 ++++++++++++++--- 2 files changed, 4472 insertions(+), 1004 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 505fae4e840f7..e455fabfe2e8d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -18177,16 +18177,38 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG, unsigned ExtOpcode = Op0.getOpcode(); SDValue A = Op0; SDValue B; + unsigned DotOpcode; if (ExtOpcode == ISD::MUL) { A = Op0.getOperand(0); B = Op0.getOperand(1); - if (A.getOpcode() != B.getOpcode() || - A.getOperand(0).getValueType() != B.getOperand(0).getValueType()) + if (A.getOperand(0).getValueType() != B.getOperand(0).getValueType()) return SDValue(); - ExtOpcode = A.getOpcode(); - } - if (ExtOpcode != ISD::ZERO_EXTEND && ExtOpcode != ISD::SIGN_EXTEND) + auto OpCodeA = A.getOpcode(); + if (OpCodeA != ISD::ZERO_EXTEND && OpCodeA != ISD::SIGN_EXTEND) + return SDValue(); + + auto OpCodeB = B.getOpcode(); + if (OpCodeB != ISD::ZERO_EXTEND && OpCodeB != ISD::SIGN_EXTEND) + return SDValue(); + + if (OpCodeA == OpCodeB) { + DotOpcode = + OpCodeA == ISD::ZERO_EXTEND ? AArch64ISD::UDOT : AArch64ISD::SDOT; + } else { + // Check USDOT support support + if (!ST->hasMatMulInt8()) + return SDValue(); + DotOpcode = AArch64ISD::USDOT; + if (OpCodeA == ISD::SIGN_EXTEND) + std::swap(A, B); + } + } else if (ExtOpcode == ISD::ZERO_EXTEND) { + DotOpcode = AArch64ISD::UDOT; + } else if (ExtOpcode == ISD::SIGN_EXTEND) { + DotOpcode = AArch64ISD::SDOT; + } else { return SDValue(); + } EVT Op0VT = A.getOperand(0).getValueType(); bool IsValidElementCount = Op0VT.getVectorNumElements() % 8 == 0; @@ -18212,8 +18234,6 @@ static SDValue performVecReduceAddCombine(SDNode *N, SelectionDAG &DAG, NumOfVecReduce = Op0VT.getVectorNumElements() / 8; TargetType = MVT::v2i32; } - auto DotOpcode = - (ExtOpcode == ISD::ZERO_EXTEND) ? AArch64ISD::UDOT : AArch64ISD::SDOT; // Handle the case where we need to generate only one Dot operation. if (NumOfVecReduce == 1) { SDValue Zeros = DAG.getConstant(0, DL, TargetType); diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll index c345c1e50bbbb..05ac2956da00c 100644 --- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll +++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll @@ -1,22 +1,28 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD -; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI +; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD +; RUN: llc -mtriple aarch64-linux-gnu -mattr=+dotprod,+i8mm -global-isel -global-isel-abort=2 < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI ; CHECK-GI: warning: Instruction selection used fallback path for test_udot_v5i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v5i8_nomla ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8_nomla ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8_nomla ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8 +; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8_double declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>) declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>) @@ -290,6 +296,128 @@ entry: ret i32 %x } +define i32 @test_usdot_v4i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_v4i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ldr s0, [x0] +; CHECK-SD-NEXT: ldr s1, [x1] +; CHECK-SD-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-SD-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-SD-NEXT: smull v0.4s, v1.4h, v0.4h +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v4i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr w8, [x0] +; CHECK-GI-NEXT: ldr w9, [x1] +; CHECK-GI-NEXT: fmov s0, w8 +; CHECK-GI-NEXT: fmov s2, w9 +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: sxtb w9, w9 +; CHECK-GI-NEXT: mov b1, v0.b[1] +; CHECK-GI-NEXT: mov b3, v0.b[2] +; CHECK-GI-NEXT: mov b5, v2.b[2] +; CHECK-GI-NEXT: mov b4, v0.b[3] +; CHECK-GI-NEXT: mov b0, v2.b[1] +; CHECK-GI-NEXT: mov b6, v2.b[3] +; CHECK-GI-NEXT: fmov s2, w9 +; CHECK-GI-NEXT: fmov w10, s1 +; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: fmov s1, w8 +; CHECK-GI-NEXT: fmov w13, s5 +; CHECK-GI-NEXT: fmov w8, s4 +; CHECK-GI-NEXT: fmov w12, s0 +; CHECK-GI-NEXT: uxtb w10, w10 +; CHECK-GI-NEXT: uxtb w11, w11 +; CHECK-GI-NEXT: sxtb w13, w13 +; CHECK-GI-NEXT: uxtb w8, w8 +; CHECK-GI-NEXT: sxtb w12, w12 +; CHECK-GI-NEXT: mov v1.h[1], w10 +; CHECK-GI-NEXT: fmov w10, s6 +; CHECK-GI-NEXT: fmov s0, w11 +; CHECK-GI-NEXT: fmov s3, w13 +; CHECK-GI-NEXT: mov v2.h[1], w12 +; CHECK-GI-NEXT: sxtb w10, w10 +; CHECK-GI-NEXT: mov v0.h[1], w8 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mov v3.h[1], w10 +; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: mov v1.d[1], v0.d[0] +; CHECK-GI-NEXT: mov v2.d[1], v3.d[0] +; CHECK-GI-NEXT: mul v0.4s, v2.4s, v1.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ret +entry: + %0 = load <4 x i8>, ptr %a + %1 = zext <4 x i8> %0 to <4 x i32> + %2 = load <4 x i8>, ptr %b + %3 = sext <4 x i8> %2 to <4 x i32> + %4 = mul nsw <4 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v4i8_double(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v4i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-SD-NEXT: bic v2.4h, #255, lsl #8 +; CHECK-SD-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-SD-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-SD-NEXT: shl v3.4s, v3.4s, #24 +; CHECK-SD-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-SD-NEXT: shl v1.4s, v1.4s, #24 +; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-SD-NEXT: sshr v3.4s, v3.4s, #24 +; CHECK-SD-NEXT: sshr v1.4s, v1.4s, #24 +; CHECK-SD-NEXT: mul v2.4s, v2.4s, v3.4s +; CHECK-SD-NEXT: mla v2.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: addv s0, v2.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v4i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: movi v4.2d, #0x0000ff000000ff +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: shl v1.4s, v1.4s, #24 +; CHECK-GI-NEXT: shl v3.4s, v3.4s, #24 +; CHECK-GI-NEXT: and v0.16b, v0.16b, v4.16b +; CHECK-GI-NEXT: and v2.16b, v2.16b, v4.16b +; CHECK-GI-NEXT: sshr v1.4s, v1.4s, #24 +; CHECK-GI-NEXT: sshr v3.4s, v3.4s, #24 +; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: mul v1.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %az = zext <4 x i8> %a to <4 x i32> + %bz = sext <4 x i8> %b to <4 x i32> + %m1 = mul nuw nsw <4 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m1) + %cz = zext <4 x i8> %c to <4 x i32> + %dz = sext <4 x i8> %d to <4 x i32> + %m2 = mul nuw nsw <4 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + define i32 @test_udot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v5i8: ; CHECK: // %bb.0: // %entry @@ -414,6 +542,65 @@ entry: ret i32 %x } +define i32 @test_usdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_usdot_v5i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldr d0, [x0] +; CHECK-NEXT: ldr d1, [x1] +; CHECK-NEXT: movi v3.2d, #0000000000000000 +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h +; CHECK-NEXT: mov v3.s[0], v2.s[0] +; CHECK-NEXT: smlal v3.4s, v1.4h, v0.4h +; CHECK-NEXT: addv s0, v3.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret +entry: + %0 = load <5 x i8>, ptr %a + %1 = zext <5 x i8> %0 to <5 x i32> + %2 = load <5 x i8>, ptr %b + %3 = sext <5 x i8> %2 to <5 x i32> + %4 = mul nsw <5 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) { +; CHECK-LABEL: test_usdot_v5i8_double: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v2.8h, v2.8b, #0 +; CHECK-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: movi v6.2d, #0000000000000000 +; CHECK-NEXT: smull2 v4.4s, v0.8h, v1.8h +; CHECK-NEXT: smull2 v7.4s, v2.8h, v3.8h +; CHECK-NEXT: mov v6.s[0], v4.s[0] +; CHECK-NEXT: mov v5.s[0], v7.s[0] +; CHECK-NEXT: smlal v6.4s, v0.4h, v1.4h +; CHECK-NEXT: smlal v5.4s, v2.4h, v3.4h +; CHECK-NEXT: add v0.4s, v6.4s, v5.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret +entry: + %az = zext <5 x i8> %a to <5 x i32> + %bz = sext <5 x i8> %b to <5 x i32> + %m1 = mul nuw nsw <5 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1) + %cz = zext <5 x i8> %c to <5 x i32> + %dz = sext <5 x i8> %d to <5 x i32> + %m2 = mul nuw nsw <5 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + + define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) { ; CHECK-LABEL: test_udot_v8i8: ; CHECK: // %bb.0: // %entry @@ -508,6 +695,77 @@ entry: ret i32 %2 } +define i32 @test_usdot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) { +; CHECK-SD-LABEL: test_usdot_v8i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr d1, [x0] +; CHECK-SD-NEXT: ldr d2, [x1] +; CHECK-SD-NEXT: usdot v0.2s, v1.8b, v2.8b +; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v8i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr d0, [x0] +; CHECK-GI-NEXT: ldr d1, [x1] +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll2 v2.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll2 v3.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v2.4s, v3.4s, v2.4s +; CHECK-GI-NEXT: mla v2.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: addv s0, v2.4s +; CHECK-GI-NEXT: fmov w0, s0 +; CHECK-GI-NEXT: ret +entry: + %0 = load <8 x i8>, ptr %a + %1 = zext <8 x i8> %0 to <8 x i32> + %2 = load <8 x i8>, ptr %b + %3 = sext <8 x i8> %2 to <8 x i32> + %4 = mul nsw <8 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4) + ret i32 %5 +} + +define i32 @test_usdot_swapped_operands_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) { +; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr d1, [x0] +; CHECK-SD-NEXT: ldr d2, [x1] +; CHECK-SD-NEXT: usdot v0.2s, v2.8b, v1.8b +; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr d0, [x0] +; CHECK-GI-NEXT: ldr d1, [x1] +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v2.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v2.4s, v3.4s, v2.4s +; CHECK-GI-NEXT: mla v2.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: addv s0, v2.4s +; CHECK-GI-NEXT: fmov w0, s0 +; CHECK-GI-NEXT: ret +entry: + %0 = load <8 x i8>, ptr %a + %1 = sext <8 x i8> %0 to <8 x i32> + %2 = load <8 x i8>, ptr %b + %3 = zext <8 x i8> %2 to <8 x i32> + %4 = mul nsw <8 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %4) + ret i32 %5 +} define i32 @test_udot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v16i8: @@ -587,6 +845,101 @@ entry: ret i32 %2 } +define i32 @test_usdot_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_v16i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q1, [x0] +; CHECK-SD-NEXT: ldr q2, [x1] +; CHECK-SD-NEXT: usdot v0.4s, v1.16b, v2.16b +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v16i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr q0, [x0] +; CHECK-GI-NEXT: ldr q1, [x1] +; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v3.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: ushll2 v4.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll2 v5.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v3.8h, #0 +; CHECK-GI-NEXT: sshll2 v7.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v4.4s, v6.4s, v4.4s +; CHECK-GI-NEXT: mul v5.4s, v7.4s, v5.4s +; CHECK-GI-NEXT: mla v4.4s, v3.4s, v2.4s +; CHECK-GI-NEXT: mla v5.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: add v0.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ret +entry: + %0 = load <16 x i8>, ptr %a + %1 = zext <16 x i8> %0 to <16 x i32> + %2 = load <16 x i8>, ptr %b + %3 = sext <16 x i8> %2 to <16 x i32> + %4 = mul nsw <16 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_swapped_operands_v16i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q1, [x0] +; CHECK-SD-NEXT: ldr q2, [x1] +; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v1.16b +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr q0, [x0] +; CHECK-GI-NEXT: ldr q1, [x1] +; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: sshll2 v4.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll2 v5.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll2 v6.4s, v3.8h, #0 +; CHECK-GI-NEXT: ushll2 v7.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v4.4s, v6.4s, v4.4s +; CHECK-GI-NEXT: mul v5.4s, v7.4s, v5.4s +; CHECK-GI-NEXT: mla v4.4s, v3.4s, v2.4s +; CHECK-GI-NEXT: mla v5.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: add v0.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ret +entry: + %0 = load <16 x i8>, ptr %a + %1 = sext <16 x i8> %0 to <16 x i32> + %2 = load <16 x i8>, ptr %b + %3 = zext <16 x i8> %2 to <16 x i32> + %4 = mul nsw <16 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} define i32 @test_udot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) { ; CHECK-SD-LABEL: test_udot_v8i8_double: @@ -860,19 +1213,253 @@ entry: ret i32 %x } -define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-SD-LABEL: test_udot_v24i8: + +define i32 @test_usdot_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v8i8_double: ; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 -; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 -; CHECK-SD-NEXT: ldr q2, [x0] -; CHECK-SD-NEXT: ldr q3, [x1] -; CHECK-SD-NEXT: ldr d4, [x0, #16] -; CHECK-SD-NEXT: ldr d5, [x1, #16] -; CHECK-SD-NEXT: udot v1.2s, v5.8b, v4.8b -; CHECK-SD-NEXT: udot v0.4s, v3.16b, v2.16b -; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s -; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: usdot v5.2s, v0.8b, v1.8b +; CHECK-SD-NEXT: usdot v4.2s, v2.8b, v3.8b +; CHECK-SD-NEXT: add v0.2s, v5.2s, v4.2s +; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v8i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 +; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-GI-NEXT: ushll2 v4.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll2 v5.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll2 v6.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll2 v7.4s, v3.8h, #0 +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: mul v4.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: mul v5.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: mla v4.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: mla v5.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: addv s0, v4.4s +; CHECK-GI-NEXT: addv s1, v5.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %az = zext <8 x i8> %a to <8 x i32> + %bz = sext <8 x i8> %b to <8 x i32> + %m1 = mul nuw nsw <8 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1) + %cz = zext <8 x i8> %c to <8 x i32> + %dz = sext <8 x i8> %d to <8 x i32> + %m2 = mul nuw nsw <8 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_usdot_swapped_operands_v8i8_double(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_swapped_operands_v8i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: usdot v5.2s, v1.8b, v0.8b +; CHECK-SD-NEXT: usdot v4.2s, v3.8b, v2.8b +; CHECK-SD-NEXT: add v0.2s, v5.2s, v4.2s +; CHECK-SD-NEXT: addp v0.2s, v0.2s, v0.2s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_swapped_operands_v8i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll v2.8h, v2.8b, #0 +; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 +; CHECK-GI-NEXT: sshll2 v4.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll2 v5.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll2 v7.4s, v3.8h, #0 +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: mul v4.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: mul v5.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: mla v4.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: mla v5.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: addv s0, v4.4s +; CHECK-GI-NEXT: addv s1, v5.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %az = sext <8 x i8> %a to <8 x i32> + %bz = zext <8 x i8> %b to <8 x i32> + %m1 = mul nuw nsw <8 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m1) + %cz = sext <8 x i8> %c to <8 x i32> + %dz = zext <8 x i8> %d to <8 x i32> + %m2 = mul nuw nsw <8 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_usdot_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v16i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: usdot v5.4s, v0.16b, v1.16b +; CHECK-SD-NEXT: usdot v4.4s, v2.16b, v3.16b +; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v16i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v5.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: ushll v6.8h, v2.8b, #0 +; CHECK-GI-NEXT: sshll v7.8h, v3.8b, #0 +; CHECK-GI-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-GI-NEXT: sshll2 v3.8h, v3.16b, #0 +; CHECK-GI-NEXT: ushll2 v16.4s, v4.8h, #0 +; CHECK-GI-NEXT: ushll2 v17.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll2 v18.4s, v5.8h, #0 +; CHECK-GI-NEXT: sshll2 v19.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll2 v20.4s, v6.8h, #0 +; CHECK-GI-NEXT: sshll2 v21.4s, v7.8h, #0 +; CHECK-GI-NEXT: ushll2 v22.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll2 v23.4s, v3.8h, #0 +; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0 +; CHECK-GI-NEXT: mul v16.4s, v16.4s, v18.4s +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v5.4s, v5.4h, #0 +; CHECK-GI-NEXT: mul v17.4s, v17.4s, v19.4s +; CHECK-GI-NEXT: mul v18.4s, v20.4s, v21.4s +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v19.4s, v22.4s, v23.4s +; CHECK-GI-NEXT: ushll v6.4s, v6.4h, #0 +; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll v7.4s, v7.4h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: mla v16.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: mla v17.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: mla v18.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: mla v19.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s +; CHECK-GI-NEXT: add v1.4s, v18.4s, v19.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %az = zext <16 x i8> %a to <16 x i32> + %bz = sext <16 x i8> %b to <16 x i32> + %m1 = mul nuw nsw <16 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1) + %cz = zext <16 x i8> %c to <16 x i32> + %dz = sext <16 x i8> %d to <16 x i32> + %m2 = mul nuw nsw <16 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + + +define i32 @test_usdot_swapped_operands_v16i8_double(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_swapped_operands_v16i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: usdot v5.4s, v1.16b, v0.16b +; CHECK-SD-NEXT: usdot v4.4s, v3.16b, v2.16b +; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_swapped_operands_v16i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sshll v4.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: sshll v6.8h, v2.8b, #0 +; CHECK-GI-NEXT: ushll v7.8h, v3.8b, #0 +; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-GI-NEXT: ushll2 v3.8h, v3.16b, #0 +; CHECK-GI-NEXT: sshll2 v16.4s, v4.8h, #0 +; CHECK-GI-NEXT: sshll2 v17.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll2 v18.4s, v5.8h, #0 +; CHECK-GI-NEXT: ushll2 v19.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll2 v20.4s, v6.8h, #0 +; CHECK-GI-NEXT: ushll2 v21.4s, v7.8h, #0 +; CHECK-GI-NEXT: sshll2 v22.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll2 v23.4s, v3.8h, #0 +; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0 +; CHECK-GI-NEXT: mul v16.4s, v16.4s, v18.4s +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0 +; CHECK-GI-NEXT: mul v17.4s, v17.4s, v19.4s +; CHECK-GI-NEXT: mul v18.4s, v20.4s, v21.4s +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v19.4s, v22.4s, v23.4s +; CHECK-GI-NEXT: sshll v6.4s, v6.4h, #0 +; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll v7.4s, v7.4h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: mla v16.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: mla v17.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: mla v18.4s, v6.4s, v7.4s +; CHECK-GI-NEXT: mla v19.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s +; CHECK-GI-NEXT: add v1.4s, v18.4s, v19.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %az = sext <16 x i8> %a to <16 x i32> + %bz = zext <16 x i8> %b to <16 x i32> + %m1 = mul nuw nsw <16 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m1) + %cz = sext <16 x i8> %c to <16 x i32> + %dz = zext <16 x i8> %d to <16 x i32> + %m2 = mul nuw nsw <16 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_udot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_udot_v24i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q2, [x0] +; CHECK-SD-NEXT: ldr q3, [x1] +; CHECK-SD-NEXT: ldr d4, [x0, #16] +; CHECK-SD-NEXT: ldr d5, [x1, #16] +; CHECK-SD-NEXT: udot v1.2s, v5.8b, v4.8b +; CHECK-SD-NEXT: udot v0.4s, v3.16b, v2.16b +; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s +; CHECK-SD-NEXT: addv s0, v0.4s ; CHECK-SD-NEXT: fmov w8, s1 ; CHECK-SD-NEXT: fmov w9, s0 ; CHECK-SD-NEXT: add w8, w9, w8 @@ -1658,91 +2245,661 @@ entry: ret i32 %x } - -define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_udot_v25i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q3, q0, [x1] -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: ldp q2, q1, [x0] -; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b -; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b -; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b -; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b -; CHECK-NEXT: ushll v3.4s, v4.4h, #0 -; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h -; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h -; CHECK-NEXT: mov v5.s[0], v3.s[0] -; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret +define i32 @test_usdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_v24i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q2, [x0] +; CHECK-SD-NEXT: ldr q3, [x1] +; CHECK-SD-NEXT: ldr d4, [x1, #16] +; CHECK-SD-NEXT: ldr d5, [x0, #16] +; CHECK-SD-NEXT: usdot v1.2s, v5.8b, v4.8b +; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v3.16b +; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s1 +; CHECK-SD-NEXT: fmov w9, s0 +; CHECK-SD-NEXT: add w8, w9, w8 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v24i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldr q0, [x0] +; CHECK-GI-NEXT: ldr d1, [x0, #16] +; CHECK-GI-NEXT: ldr q2, [x1] +; CHECK-GI-NEXT: ldr d3, [x1, #16] +; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v5.8h, v2.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-GI-NEXT: ushll v6.4s, v4.4h, #0 +; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0 +; CHECK-GI-NEXT: ushll v7.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll v16.4s, v5.4h, #0 +; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0 +; CHECK-GI-NEXT: ushll v17.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v18.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll v19.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: mul v6.4s, v16.4s, v6.4s +; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s +; CHECK-GI-NEXT: mul v0.4s, v18.4s, v0.4s +; CHECK-GI-NEXT: mul v5.4s, v19.4s, v17.4s +; CHECK-GI-NEXT: mul v2.4s, v2.4s, v7.4s +; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s +; CHECK-GI-NEXT: addv s3, v6.4s +; CHECK-GI-NEXT: addv s4, v4.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s5, v5.4s +; CHECK-GI-NEXT: addv s2, v2.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s3 +; CHECK-GI-NEXT: fmov w9, s4 +; CHECK-GI-NEXT: fmov w10, s0 +; CHECK-GI-NEXT: fmov w11, s5 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: add w10, w10, w11 +; CHECK-GI-NEXT: fmov w11, s1 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w9, w10, w11 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ret entry: - %0 = load <25 x i8>, ptr %a - %1 = zext <25 x i8> %0 to <25 x i32> - %2 = load <25 x i8>, ptr %b - %3 = zext <25 x i8> %2 to <25 x i32> - %4 = mul nuw nsw <25 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) - %op.extra = add i32 %5, %sum + %0 = load <24 x i8>, ptr %a + %1 = zext <24 x i8> %0 to <24 x i32> + %2 = load <24 x i8>, ptr %b + %3 = sext <24 x i8> %2 to <24 x i32> + %4 = mul nsw <24 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4) + %op.extra = add nsw i32 %5, %sum ret i32 %op.extra } -define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) { -; CHECK-LABEL: test_udot_v25i8_nomla: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q1, [x0] -; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v4.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: ushll v3.4s, v3.4h, #0 -; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h -; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h -; CHECK-NEXT: mov v0.s[0], v3.s[0] -; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h -; CHECK-NEXT: add v1.4s, v1.4s, v3.4s -; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ret -entry: - %0 = load <25 x i8>, ptr %a1 - %1 = zext <25 x i8> %0 to <25 x i32> - %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1) - ret i32 %2 -} -define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_sdot_v25i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q3, q0, [x1] -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: ldp q2, q1, [x0] -; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b -; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b -; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b -; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b -; CHECK-NEXT: sshll v3.4s, v4.4h, #0 -; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h -; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h -; CHECK-NEXT: mov v5.s[0], v3.s[0] -; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret -entry: - %0 = load <25 x i8>, ptr %a - %1 = sext <25 x i8> %0 to <25 x i32> +define i32 @test_usdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v24i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 +; CHECK-SD-NEXT: .cfi_offset w29, -16 +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: ldr b1, [sp, #144] +; CHECK-SD-NEXT: add x10, sp, #152 +; CHECK-SD-NEXT: add x9, sp, #160 +; CHECK-SD-NEXT: add x8, sp, #168 +; CHECK-SD-NEXT: ldr b2, [sp, #272] +; CHECK-SD-NEXT: ld1 { v1.b }[1], [x10] +; CHECK-SD-NEXT: add x11, sp, #280 +; CHECK-SD-NEXT: ldr b3, [sp, #80] +; CHECK-SD-NEXT: mov v0.b[1], w1 +; CHECK-SD-NEXT: ldr b4, [sp, #528] +; CHECK-SD-NEXT: add x10, sp, #88 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x11] +; CHECK-SD-NEXT: add x11, sp, #536 +; CHECK-SD-NEXT: ldr b5, [sp, #336] +; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10] +; CHECK-SD-NEXT: add x10, sp, #344 +; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11] +; CHECK-SD-NEXT: add x11, sp, #176 +; CHECK-SD-NEXT: ldr b6, [sp, #656] +; CHECK-SD-NEXT: mov v0.b[2], w2 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10] +; CHECK-SD-NEXT: ldr b7, [sp, #464] +; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] +; CHECK-SD-NEXT: add x12, sp, #664 +; CHECK-SD-NEXT: add x9, sp, #472 +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] +; CHECK-SD-NEXT: add x8, sp, #96 +; CHECK-SD-NEXT: add x10, sp, #184 +; CHECK-SD-NEXT: add x12, sp, #288 +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8] +; CHECK-SD-NEXT: mov v0.b[3], w3 +; CHECK-SD-NEXT: ld1 { v1.b }[4], [x11] +; CHECK-SD-NEXT: add x8, sp, #352 +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12] +; CHECK-SD-NEXT: add x13, sp, #544 +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #672 +; CHECK-SD-NEXT: ld1 { v4.b }[2], [x13] +; CHECK-SD-NEXT: add x9, sp, #192 +; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #480 +; CHECK-SD-NEXT: mov v0.b[4], w4 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #296 +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #552 +; CHECK-SD-NEXT: add x12, sp, #200 +; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #360 +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #104 +; CHECK-SD-NEXT: add x9, sp, #560 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #368 +; CHECK-SD-NEXT: ld1 { v1.b }[7], [x12] +; CHECK-SD-NEXT: ld1 { v4.b }[4], [x9] +; CHECK-SD-NEXT: add x13, sp, #208 +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] +; CHECK-SD-NEXT: add x12, sp, #304 +; CHECK-SD-NEXT: add x8, sp, #568 +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12] +; CHECK-SD-NEXT: add x12, sp, #16 +; CHECK-SD-NEXT: add x17, sp, #376 +; CHECK-SD-NEXT: mov v0.b[6], w6 +; CHECK-SD-NEXT: ld1 { v1.b }[8], [x13] +; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8] +; CHECK-SD-NEXT: add x14, sp, #216 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x17] +; CHECK-SD-NEXT: add x13, sp, #576 +; CHECK-SD-NEXT: add x11, sp, #224 +; CHECK-SD-NEXT: add x10, sp, #232 +; CHECK-SD-NEXT: add x15, sp, #240 +; CHECK-SD-NEXT: ld1 { v1.b }[9], [x14] +; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] +; CHECK-SD-NEXT: add x13, sp, #384 +; CHECK-SD-NEXT: mov v0.b[7], w7 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x13] +; CHECK-SD-NEXT: add x13, sp, #112 +; CHECK-SD-NEXT: ld1 { v3.b }[4], [x13] +; CHECK-SD-NEXT: add x13, sp, #32 +; CHECK-SD-NEXT: add x14, sp, #584 +; CHECK-SD-NEXT: ld1 { v1.b }[10], [x11] +; CHECK-SD-NEXT: ld1 { v4.b }[7], [x14] +; CHECK-SD-NEXT: add x11, sp, #312 +; CHECK-SD-NEXT: add x14, sp, #40 +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x11] +; CHECK-SD-NEXT: add x11, sp, #592 +; CHECK-SD-NEXT: ld1 { v0.b }[8], [x12] +; CHECK-SD-NEXT: add x12, sp, #24 +; CHECK-SD-NEXT: add x16, sp, #248 +; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10] +; CHECK-SD-NEXT: ld1 { v4.b }[8], [x11] +; CHECK-SD-NEXT: add x11, sp, #400 +; CHECK-SD-NEXT: add x9, sp, #256 +; CHECK-SD-NEXT: add x8, sp, #264 +; CHECK-SD-NEXT: add x10, sp, #72 +; CHECK-SD-NEXT: ld1 { v0.b }[9], [x12] +; CHECK-SD-NEXT: add x12, sp, #392 +; CHECK-SD-NEXT: movi v16.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x12] +; CHECK-SD-NEXT: add x12, sp, #48 +; CHECK-SD-NEXT: ld1 { v1.b }[12], [x15] +; CHECK-SD-NEXT: add x15, sp, #120 +; CHECK-SD-NEXT: movi v17.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v18.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v0.b }[10], [x13] +; CHECK-SD-NEXT: ld1 { v3.b }[5], [x15] +; CHECK-SD-NEXT: add x15, sp, #408 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11] +; CHECK-SD-NEXT: add x13, sp, #56 +; CHECK-SD-NEXT: ld1 { v1.b }[13], [x16] +; CHECK-SD-NEXT: add x11, sp, #64 +; CHECK-SD-NEXT: add x16, sp, #616 +; CHECK-SD-NEXT: movi v19.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v0.b }[11], [x14] +; CHECK-SD-NEXT: add x14, sp, #600 +; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x15] +; CHECK-SD-NEXT: add x15, sp, #608 +; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9] +; CHECK-SD-NEXT: add x9, sp, #488 +; CHECK-SD-NEXT: add x14, sp, #320 +; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x14] +; CHECK-SD-NEXT: ld1 { v4.b }[10], [x15] +; CHECK-SD-NEXT: add x14, sp, #624 +; CHECK-SD-NEXT: add x9, sp, #688 +; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8] +; CHECK-SD-NEXT: add x8, sp, #432 +; CHECK-SD-NEXT: add x12, sp, #328 +; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13] +; CHECK-SD-NEXT: add x13, sp, #416 +; CHECK-SD-NEXT: ld1 { v2.b }[7], [x12] +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x13] +; CHECK-SD-NEXT: ld1 { v4.b }[11], [x16] +; CHECK-SD-NEXT: add x16, sp, #680 +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x16] +; CHECK-SD-NEXT: add x13, sp, #632 +; CHECK-SD-NEXT: add x12, sp, #504 +; CHECK-SD-NEXT: ld1 { v0.b }[14], [x11] +; CHECK-SD-NEXT: add x11, sp, #424 +; CHECK-SD-NEXT: add x15, sp, #128 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x11] +; CHECK-SD-NEXT: ld1 { v4.b }[12], [x14] +; CHECK-SD-NEXT: add x11, sp, #696 +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9] +; CHECK-SD-NEXT: ld1 { v3.b }[6], [x15] +; CHECK-SD-NEXT: add x9, sp, #640 +; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] +; CHECK-SD-NEXT: add x10, sp, #496 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10] +; CHECK-SD-NEXT: ld1 { v4.b }[13], [x13] +; CHECK-SD-NEXT: add x10, sp, #440 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x11] +; CHECK-SD-NEXT: add x11, sp, #512 +; CHECK-SD-NEXT: add x8, sp, #136 +; CHECK-SD-NEXT: usdot v17.4s, v0.16b, v1.16b +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x10] +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x12] +; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] +; CHECK-SD-NEXT: add x9, sp, #448 +; CHECK-SD-NEXT: add x10, sp, #704 +; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10] +; CHECK-SD-NEXT: add x8, sp, #648 +; CHECK-SD-NEXT: add x10, sp, #520 +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11] +; CHECK-SD-NEXT: ld1 { v4.b }[15], [x8] +; CHECK-SD-NEXT: add x8, sp, #456 +; CHECK-SD-NEXT: add x9, sp, #712 +; CHECK-SD-NEXT: usdot v19.2s, v3.8b, v2.8b +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-SD-NEXT: addv s0, v17.4s +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10] +; CHECK-SD-NEXT: addp v1.2s, v19.2s, v19.2s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: usdot v16.4s, v5.16b, v4.16b +; CHECK-SD-NEXT: usdot v18.2s, v7.8b, v6.8b +; CHECK-SD-NEXT: fmov w9, s1 +; CHECK-SD-NEXT: addv s2, v16.4s +; CHECK-SD-NEXT: addp v3.2s, v18.2s, v18.2s +; CHECK-SD-NEXT: add w8, w8, w9 +; CHECK-SD-NEXT: fmov w10, s2 +; CHECK-SD-NEXT: fmov w11, s3 +; CHECK-SD-NEXT: add w9, w10, w11 +; CHECK-SD-NEXT: add w0, w8, w9 +; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v24i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 +; CHECK-GI-NEXT: .cfi_offset w29, -16 +; CHECK-GI-NEXT: ldr w9, [sp, #16] +; CHECK-GI-NEXT: ldr w8, [sp, #24] +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: ldr w11, [sp, #208] +; CHECK-GI-NEXT: ldr w13, [sp, #32] +; CHECK-GI-NEXT: ldr w12, [sp, #144] +; CHECK-GI-NEXT: fmov s1, w9 +; CHECK-GI-NEXT: ldr w9, [sp, #80] +; CHECK-GI-NEXT: ldr w10, [sp, #152] +; CHECK-GI-NEXT: fmov s4, w11 +; CHECK-GI-NEXT: fmov s3, w12 +; CHECK-GI-NEXT: ldr w11, [sp, #160] +; CHECK-GI-NEXT: fmov s2, w9 +; CHECK-GI-NEXT: ldr w9, [sp, #216] +; CHECK-GI-NEXT: ldr w14, [sp, #128] +; CHECK-GI-NEXT: mov v1.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #88] +; CHECK-GI-NEXT: ldr w12, [sp, #240] +; CHECK-GI-NEXT: mov v4.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #96] +; CHECK-GI-NEXT: mov v3.b[1], w10 +; CHECK-GI-NEXT: mov v2.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #40] +; CHECK-GI-NEXT: ldr w10, [sp, #272] +; CHECK-GI-NEXT: ldr w15, [sp, #312] +; CHECK-GI-NEXT: mov v0.b[1], w1 +; CHECK-GI-NEXT: mov v1.b[2], w13 +; CHECK-GI-NEXT: fmov s5, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #48] +; CHECK-GI-NEXT: mov v3.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #232] +; CHECK-GI-NEXT: ldr w13, [sp, #72] +; CHECK-GI-NEXT: mov v2.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #280] +; CHECK-GI-NEXT: mov v0.b[2], w2 +; CHECK-GI-NEXT: mov v1.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #224] +; CHECK-GI-NEXT: mov v5.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #168] +; CHECK-GI-NEXT: mov v4.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #104] +; CHECK-GI-NEXT: mov v3.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #64] +; CHECK-GI-NEXT: mov v0.b[3], w3 +; CHECK-GI-NEXT: mov v2.b[3], w8 +; CHECK-GI-NEXT: mov v1.b[4], w10 +; CHECK-GI-NEXT: ldr w8, [sp, #288] +; CHECK-GI-NEXT: ldr w10, [sp, #56] +; CHECK-GI-NEXT: mov v4.b[3], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #112] +; CHECK-GI-NEXT: mov v5.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #176] +; CHECK-GI-NEXT: mov v0.b[4], w4 +; CHECK-GI-NEXT: mov v2.b[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #120] +; CHECK-GI-NEXT: mov v1.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #296] +; CHECK-GI-NEXT: mov v3.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #136] +; CHECK-GI-NEXT: mov v4.b[4], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #248] +; CHECK-GI-NEXT: mov v5.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #184] +; CHECK-GI-NEXT: mov v0.b[5], w5 +; CHECK-GI-NEXT: mov v2.b[5], w11 +; CHECK-GI-NEXT: mov v1.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #304] +; CHECK-GI-NEXT: mov v3.b[5], w10 +; CHECK-GI-NEXT: ldr w11, [sp, #192] +; CHECK-GI-NEXT: ldr w10, [sp, #200] +; CHECK-GI-NEXT: mov v4.b[5], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #264] +; CHECK-GI-NEXT: mov v5.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #256] +; CHECK-GI-NEXT: mov v0.b[6], w6 +; CHECK-GI-NEXT: mov v2.b[6], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #336] +; CHECK-GI-NEXT: mov v1.b[7], w13 +; CHECK-GI-NEXT: mov v3.b[6], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #344] +; CHECK-GI-NEXT: ldr w13, [sp, #320] +; CHECK-GI-NEXT: fmov s6, w14 +; CHECK-GI-NEXT: ldr w14, [sp, #400] +; CHECK-GI-NEXT: mov v4.b[6], w9 +; CHECK-GI-NEXT: mov v5.b[5], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #408] +; CHECK-GI-NEXT: ldr w9, [sp, #328] +; CHECK-GI-NEXT: fmov s7, w14 +; CHECK-GI-NEXT: ldr w14, [sp, #352] +; CHECK-GI-NEXT: mov v2.b[7], w8 +; CHECK-GI-NEXT: mov v6.b[1], w11 +; CHECK-GI-NEXT: mov v3.b[7], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #416] +; CHECK-GI-NEXT: mov v4.b[7], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #472] +; CHECK-GI-NEXT: ldr w11, [sp, #360] +; CHECK-GI-NEXT: mov v7.b[1], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #464] +; CHECK-GI-NEXT: mov v5.b[6], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #536] +; CHECK-GI-NEXT: ldr w8, [sp, #368] +; CHECK-GI-NEXT: mov v0.b[7], w7 +; CHECK-GI-NEXT: mov v6.b[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #528] +; CHECK-GI-NEXT: fmov s16, w15 +; CHECK-GI-NEXT: ldr w15, [sp, #600] +; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: fmov s17, w14 +; CHECK-GI-NEXT: mov v7.b[2], w10 +; CHECK-GI-NEXT: ldr w14, [sp, #592] +; CHECK-GI-NEXT: mov v16.b[1], w12 +; CHECK-GI-NEXT: mov v5.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #480] +; CHECK-GI-NEXT: fmov s18, w14 +; CHECK-GI-NEXT: ldr w14, [sp, #544] +; CHECK-GI-NEXT: mov v6.b[3], w11 +; CHECK-GI-NEXT: mov v17.b[1], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #424] +; CHECK-GI-NEXT: ldr w11, [sp, #608] +; CHECK-GI-NEXT: ldr w10, [sp, #376] +; CHECK-GI-NEXT: ldr w12, [sp, #384] +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: mov v7.b[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #656] +; CHECK-GI-NEXT: mov v18.b[1], w15 +; CHECK-GI-NEXT: mov v16.b[2], w9 +; CHECK-GI-NEXT: mov v6.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #616] +; CHECK-GI-NEXT: fmov s19, w13 +; CHECK-GI-NEXT: mov v17.b[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #664] +; CHECK-GI-NEXT: ldr w13, [sp, #432] +; CHECK-GI-NEXT: ushll v20.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: mov v18.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #488] +; CHECK-GI-NEXT: sshll v21.4s, v3.4h, #0 +; CHECK-GI-NEXT: mov v19.b[1], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #552] +; CHECK-GI-NEXT: mov v7.b[4], w13 +; CHECK-GI-NEXT: mov v16.b[3], w11 +; CHECK-GI-NEXT: ldr w13, [sp, #496] +; CHECK-GI-NEXT: ldr w11, [sp, #440] +; CHECK-GI-NEXT: mov v17.b[3], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #672] +; CHECK-GI-NEXT: mov v6.b[5], w10 +; CHECK-GI-NEXT: mov v18.b[3], w8 +; CHECK-GI-NEXT: ldr w10, [sp, #624] +; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: mov v19.b[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #560] +; CHECK-GI-NEXT: mov v7.b[5], w11 +; CHECK-GI-NEXT: mov v16.b[4], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #680] +; CHECK-GI-NEXT: ldr w11, [sp, #504] +; CHECK-GI-NEXT: mov v17.b[4], w14 +; CHECK-GI-NEXT: mov v6.b[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #632] +; CHECK-GI-NEXT: mov v18.b[4], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #512] +; CHECK-GI-NEXT: ldr w9, [sp, #392] +; CHECK-GI-NEXT: mov v19.b[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #568] +; CHECK-GI-NEXT: ldr w14, [sp, #584] +; CHECK-GI-NEXT: mov v16.b[5], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #688] +; CHECK-GI-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-GI-NEXT: mov v17.b[5], w13 +; CHECK-GI-NEXT: ldr w15, [sp, #448] +; CHECK-GI-NEXT: mul v20.4s, v20.4s, v21.4s +; CHECK-GI-NEXT: mov v18.b[5], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #576] +; CHECK-GI-NEXT: mul v0.4s, v0.4s, v3.4s +; CHECK-GI-NEXT: mov v19.b[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #640] +; CHECK-GI-NEXT: mov v6.b[7], w9 +; CHECK-GI-NEXT: mov v16.b[6], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #696] +; CHECK-GI-NEXT: mov v7.b[6], w15 +; CHECK-GI-NEXT: mov v17.b[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #648] +; CHECK-GI-NEXT: ushll v22.4s, v1.4h, #0 +; CHECK-GI-NEXT: mov v18.b[6], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #704] +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: mov v19.b[5], w10 +; CHECK-GI-NEXT: sshll v21.4s, v4.4h, #0 +; CHECK-GI-NEXT: sshll2 v4.4s, v4.8h, #0 +; CHECK-GI-NEXT: ldr w8, [sp, #456] +; CHECK-GI-NEXT: ldr w13, [sp, #520] +; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 +; CHECK-GI-NEXT: mov v17.b[7], w14 +; CHECK-GI-NEXT: ldr w10, [sp, #712] +; CHECK-GI-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-GI-NEXT: mov v18.b[7], w12 +; CHECK-GI-NEXT: mul v1.4s, v1.4s, v4.4s +; CHECK-GI-NEXT: addv s4, v20.4s +; CHECK-GI-NEXT: mov v19.b[6], w11 +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0 +; CHECK-GI-NEXT: mov v7.b[7], w8 +; CHECK-GI-NEXT: mov v16.b[7], w13 +; CHECK-GI-NEXT: ushll v23.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0 +; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v5.4h, #0 +; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0 +; CHECK-GI-NEXT: mul v21.4s, v22.4s, v21.4s +; CHECK-GI-NEXT: fmov w8, s4 +; CHECK-GI-NEXT: mov v19.b[7], w10 +; CHECK-GI-NEXT: sshll v4.8h, v18.8b, #0 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0 +; CHECK-GI-NEXT: sshll v18.4s, v17.4h, #0 +; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0 +; CHECK-GI-NEXT: mul v2.4s, v2.4s, v5.4s +; CHECK-GI-NEXT: ushll v5.8h, v7.8b, #0 +; CHECK-GI-NEXT: ushll v7.8h, v16.8b, #0 +; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0 +; CHECK-GI-NEXT: mul v3.4s, v23.4s, v3.4s +; CHECK-GI-NEXT: sshll2 v22.4s, v4.8h, #0 +; CHECK-GI-NEXT: sshll v16.8h, v19.8b, #0 +; CHECK-GI-NEXT: mul v0.4s, v0.4s, v18.4s +; CHECK-GI-NEXT: addv s18, v21.4s +; CHECK-GI-NEXT: ushll2 v19.4s, v5.8h, #0 +; CHECK-GI-NEXT: ushll v20.4s, v7.4h, #0 +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: mul v6.4s, v6.4s, v17.4s +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0 +; CHECK-GI-NEXT: sshll v23.4s, v16.4h, #0 +; CHECK-GI-NEXT: fmov w9, s18 +; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0 +; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0 +; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0 +; CHECK-GI-NEXT: mul v17.4s, v19.4s, v22.4s +; CHECK-GI-NEXT: addv s3, v3.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s2, v2.4s +; CHECK-GI-NEXT: mul v19.4s, v20.4s, v23.4s +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: addv s1, v6.4s +; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s +; CHECK-GI-NEXT: mul v5.4s, v7.4s, v16.4s +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: addv s3, v17.4s +; CHECK-GI-NEXT: fmov w11, s0 +; CHECK-GI-NEXT: addv s6, v19.4s +; CHECK-GI-NEXT: fmov w12, s1 +; CHECK-GI-NEXT: addv s4, v4.4s +; CHECK-GI-NEXT: addv s5, v5.4s +; CHECK-GI-NEXT: add w9, w9, w10 +; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: add w11, w11, w12 +; CHECK-GI-NEXT: fmov w12, s3 +; CHECK-GI-NEXT: fmov w13, s6 +; CHECK-GI-NEXT: fmov w14, s4 +; CHECK-GI-NEXT: add w9, w9, w10 +; CHECK-GI-NEXT: add w12, w12, w13 +; CHECK-GI-NEXT: fmov w13, s5 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w10, w11, w14 +; CHECK-GI-NEXT: add w11, w12, w13 +; CHECK-GI-NEXT: add w9, w10, w11 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-GI-NEXT: ret +entry: + %az = zext <24 x i8> %a to <24 x i32> + %bz = sext <24 x i8> %b to <24 x i32> + %m1 = mul nuw nsw <24 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1) + %cz = zext <24 x i8> %c to <24 x i32> + %dz = sext <24 x i8> %d to <24 x i32> + %m2 = mul nuw nsw <24 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_udot_v25i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q3, q0, [x1] +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b +; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b +; CHECK-NEXT: ushll v3.4s, v4.4h, #0 +; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h +; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v5.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret +entry: + %0 = load <25 x i8>, ptr %a + %1 = zext <25 x i8> %0 to <25 x i32> + %2 = load <25 x i8>, ptr %b + %3 = zext <25 x i8> %2 to <25 x i32> + %4 = mul nuw nsw <25 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) + %op.extra = add i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) { +; CHECK-LABEL: test_udot_v25i8_nomla: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: ushll v4.8h, v2.8b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h +; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h +; CHECK-NEXT: mov v0.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s +; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret +entry: + %0 = load <25 x i8>, ptr %a1 + %1 = zext <25 x i8> %0 to <25 x i32> + %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1) + ret i32 %2 +} +define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_sdot_v25i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q3, q0, [x1] +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b +; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b +; CHECK-NEXT: sshll v3.4s, v4.4h, #0 +; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h +; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v5.s[0], v3.s[0] +; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret +entry: + %0 = load <25 x i8>, ptr %a + %1 = sext <25 x i8> %0 to <25 x i32> %2 = load <25 x i8>, ptr %b %3 = sext <25 x i8> %2 to <25 x i32> %4 = mul nsw <25 x i32> %3, %1 @@ -2109,6 +3266,284 @@ entry: ret i32 %x } +define i32 @test_usdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_usdot_v25i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q2, q0, [x0] +; CHECK-NEXT: ldp q5, q1, [x1] +; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 +; CHECK-NEXT: ushll v6.8h, v2.8b, #0 +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 +; CHECK-NEXT: sshll v7.8h, v5.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h +; CHECK-NEXT: movi v4.2d, #0000000000000000 +; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h +; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h +; CHECK-NEXT: mov v4.s[0], v3.s[0] +; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0 +; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h +; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h +; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h +; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h +; CHECK-NEXT: add v0.4s, v6.4s, v4.4s +; CHECK-NEXT: add v0.4s, v0.4s, v16.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret +entry: + %0 = load <25 x i8>, ptr %a + %1 = zext <25 x i8> %0 to <25 x i32> + %2 = load <25 x i8>, ptr %b + %3 = sext <25 x i8> %2 to <25 x i32> + %4 = mul nsw <25 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) { +; CHECK-LABEL: test_usdot_v25i8_double: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: ldr b1, [sp, #16] +; CHECK-NEXT: ldr b0, [sp, #80] +; CHECK-NEXT: add x11, sp, #24 +; CHECK-NEXT: ldr b3, [sp, #216] +; CHECK-NEXT: ldr b4, [sp, #152] +; CHECK-NEXT: add x10, sp, #88 +; CHECK-NEXT: ldr b2, [sp, #280] +; CHECK-NEXT: ld1 { v1.b }[1], [x11] +; CHECK-NEXT: add x11, sp, #224 +; CHECK-NEXT: add x12, sp, #160 +; CHECK-NEXT: ldr b6, [sp, #480] +; CHECK-NEXT: ld1 { v0.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #288 +; CHECK-NEXT: ld1 { v3.b }[1], [x11] +; CHECK-NEXT: ld1 { v4.b }[1], [x12] +; CHECK-NEXT: add x12, sp, #488 +; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: add x9, sp, #96 +; CHECK-NEXT: ld1 { v6.b }[1], [x12] +; CHECK-NEXT: add x12, sp, #32 +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: ld1 { v1.b }[2], [x12] +; CHECK-NEXT: add x12, sp, #232 +; CHECK-NEXT: add x9, sp, #296 +; CHECK-NEXT: ld1 { v3.b }[2], [x12] +; CHECK-NEXT: add x12, sp, #168 +; CHECK-NEXT: ld1 { v2.b }[2], [x9] +; CHECK-NEXT: ld1 { v4.b }[2], [x12] +; CHECK-NEXT: add x12, sp, #40 +; CHECK-NEXT: add x8, sp, #104 +; CHECK-NEXT: ld1 { v1.b }[3], [x12] +; CHECK-NEXT: ld1 { v0.b }[3], [x8] +; CHECK-NEXT: add x8, sp, #304 +; CHECK-NEXT: ld1 { v2.b }[3], [x8] +; CHECK-NEXT: add x8, sp, #48 +; CHECK-NEXT: add x10, sp, #112 +; CHECK-NEXT: add x14, sp, #56 +; CHECK-NEXT: add x12, sp, #240 +; CHECK-NEXT: add x11, sp, #120 +; CHECK-NEXT: ld1 { v1.b }[4], [x8] +; CHECK-NEXT: ld1 { v0.b }[4], [x10] +; CHECK-NEXT: add x8, sp, #312 +; CHECK-NEXT: ld1 { v3.b }[3], [x12] +; CHECK-NEXT: add x12, sp, #176 +; CHECK-NEXT: ld1 { v2.b }[4], [x8] +; CHECK-NEXT: ld1 { v4.b }[3], [x12] +; CHECK-NEXT: add x12, sp, #64 +; CHECK-NEXT: ldr b18, [sp, #552] +; CHECK-NEXT: ld1 { v1.b }[5], [x14] +; CHECK-NEXT: add x16, sp, #320 +; CHECK-NEXT: ld1 { v0.b }[5], [x11] +; CHECK-NEXT: ldr b16, [sp, #352] +; CHECK-NEXT: add x8, sp, #248 +; CHECK-NEXT: ld1 { v2.b }[5], [x16] +; CHECK-NEXT: add x16, sp, #360 +; CHECK-NEXT: add x15, sp, #128 +; CHECK-NEXT: ld1 { v3.b }[4], [x8] +; CHECK-NEXT: ld1 { v1.b }[6], [x12] +; CHECK-NEXT: add x12, sp, #560 +; CHECK-NEXT: ld1 { v16.b }[1], [x16] +; CHECK-NEXT: ld1 { v18.b }[1], [x12] +; CHECK-NEXT: add x10, sp, #72 +; CHECK-NEXT: ld1 { v0.b }[6], [x15] +; CHECK-NEXT: add x15, sp, #184 +; CHECK-NEXT: add x12, sp, #568 +; CHECK-NEXT: add x13, sp, #328 +; CHECK-NEXT: add x14, sp, #256 +; CHECK-NEXT: ld1 { v4.b }[4], [x15] +; CHECK-NEXT: ld1 { v1.b }[7], [x10] +; CHECK-NEXT: add x10, sp, #368 +; CHECK-NEXT: ld1 { v18.b }[2], [x12] +; CHECK-NEXT: add x9, sp, #136 +; CHECK-NEXT: ld1 { v3.b }[5], [x14] +; CHECK-NEXT: ld1 { v2.b }[6], [x13] +; CHECK-NEXT: add x13, sp, #496 +; CHECK-NEXT: ld1 { v16.b }[2], [x10] +; CHECK-NEXT: fmov s5, w0 +; CHECK-NEXT: ld1 { v0.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #192 +; CHECK-NEXT: ld1 { v6.b }[2], [x13] +; CHECK-NEXT: add x10, sp, #576 +; CHECK-NEXT: add x11, sp, #264 +; CHECK-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #376 +; CHECK-NEXT: ld1 { v18.b }[3], [x10] +; CHECK-NEXT: ld1 { v3.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #504 +; CHECK-NEXT: ld1 { v16.b }[3], [x9] +; CHECK-NEXT: mov v5.b[1], w1 +; CHECK-NEXT: ldr b7, [sp, #144] +; CHECK-NEXT: ldr b17, [sp, #344] +; CHECK-NEXT: add x9, sp, #200 +; CHECK-NEXT: ld1 { v6.b }[3], [x11] +; CHECK-NEXT: add x10, sp, #584 +; CHECK-NEXT: ld1 { v4.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #384 +; CHECK-NEXT: ld1 { v18.b }[4], [x10] +; CHECK-NEXT: ushll v7.8h, v7.8b, #0 +; CHECK-NEXT: sshll v17.8h, v17.8b, #0 +; CHECK-NEXT: add x11, sp, #512 +; CHECK-NEXT: ld1 { v16.b }[4], [x9] +; CHECK-NEXT: mov v5.b[2], w2 +; CHECK-NEXT: ld1 { v6.b }[4], [x11] +; CHECK-NEXT: add x11, sp, #592 +; CHECK-NEXT: add x10, sp, #392 +; CHECK-NEXT: ldr b19, [sp, #680] +; CHECK-NEXT: ld1 { v18.b }[5], [x11] +; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h +; CHECK-NEXT: ldr b17, [sp, #416] +; CHECK-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-NEXT: add x10, sp, #688 +; CHECK-NEXT: add x9, sp, #424 +; CHECK-NEXT: ld1 { v19.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #600 +; CHECK-NEXT: ldr b20, [sp, #616] +; CHECK-NEXT: ld1 { v17.b }[1], [x9] +; CHECK-NEXT: add x11, sp, #400 +; CHECK-NEXT: ld1 { v18.b }[6], [x10] +; CHECK-NEXT: add x12, sp, #624 +; CHECK-NEXT: mov v5.b[3], w3 +; CHECK-NEXT: ld1 { v16.b }[6], [x11] +; CHECK-NEXT: add x11, sp, #696 +; CHECK-NEXT: ld1 { v20.b }[1], [x12] +; CHECK-NEXT: add x9, sp, #432 +; CHECK-NEXT: ld1 { v19.b }[2], [x11] +; CHECK-NEXT: add x11, sp, #608 +; CHECK-NEXT: ld1 { v17.b }[2], [x9] +; CHECK-NEXT: ld1 { v18.b }[7], [x11] +; CHECK-NEXT: add x11, sp, #632 +; CHECK-NEXT: add x10, sp, #408 +; CHECK-NEXT: ld1 { v20.b }[2], [x11] +; CHECK-NEXT: mov v5.b[4], w4 +; CHECK-NEXT: ld1 { v16.b }[7], [x10] +; CHECK-NEXT: add x10, sp, #704 +; CHECK-NEXT: add x12, sp, #440 +; CHECK-NEXT: ld1 { v19.b }[3], [x10] +; CHECK-NEXT: ld1 { v17.b }[3], [x12] +; CHECK-NEXT: add x12, sp, #640 +; CHECK-NEXT: ld1 { v20.b }[3], [x12] +; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: add x10, sp, #448 +; CHECK-NEXT: ushll v21.8h, v16.8b, #0 +; CHECK-NEXT: add x11, sp, #712 +; CHECK-NEXT: mov v5.b[5], w5 +; CHECK-NEXT: ld1 { v19.b }[4], [x11] +; CHECK-NEXT: ld1 { v17.b }[4], [x10] +; CHECK-NEXT: add x10, sp, #648 +; CHECK-NEXT: add x9, sp, #520 +; CHECK-NEXT: ld1 { v20.b }[4], [x10] +; CHECK-NEXT: ldr b22, [sp, #544] +; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h +; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h +; CHECK-NEXT: ldr b21, [sp, #744] +; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: ld1 { v6.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #456 +; CHECK-NEXT: ld1 { v19.b }[5], [x11] +; CHECK-NEXT: ld1 { v17.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #656 +; CHECK-NEXT: mov v5.b[6], w6 +; CHECK-NEXT: ushll v22.8h, v22.8b, #0 +; CHECK-NEXT: sshll v21.8h, v21.8b, #0 +; CHECK-NEXT: ld1 { v20.b }[5], [x9] +; CHECK-NEXT: add x10, sp, #528 +; CHECK-NEXT: add x11, sp, #728 +; CHECK-NEXT: ld1 { v6.b }[6], [x10] +; CHECK-NEXT: ld1 { v19.b }[6], [x11] +; CHECK-NEXT: add x10, sp, #464 +; CHECK-NEXT: add x11, sp, #664 +; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h +; CHECK-NEXT: movi v22.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v17.b }[6], [x10] +; CHECK-NEXT: ld1 { v20.b }[6], [x11] +; CHECK-NEXT: mov v5.b[7], w7 +; CHECK-NEXT: add x9, sp, #536 +; CHECK-NEXT: add x10, sp, #736 +; CHECK-NEXT: add x11, sp, #208 +; CHECK-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-NEXT: ld1 { v19.b }[7], [x10] +; CHECK-NEXT: ld1 { v4.b }[7], [x11] +; CHECK-NEXT: add x9, sp, #472 +; CHECK-NEXT: add x10, sp, #672 +; CHECK-NEXT: add x8, sp, #336 +; CHECK-NEXT: ld1 { v17.b }[7], [x9] +; CHECK-NEXT: ld1 { v20.b }[7], [x10] +; CHECK-NEXT: mov v22.s[0], v21.s[0] +; CHECK-NEXT: movi v21.2d, #0000000000000000 +; CHECK-NEXT: ushll v5.8h, v5.8b, #0 +; CHECK-NEXT: ushll v6.8h, v6.8b, #0 +; CHECK-NEXT: sshll v19.8h, v19.8b, #0 +; CHECK-NEXT: ld1 { v2.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #272 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: ushll v17.8h, v17.8b, #0 +; CHECK-NEXT: sshll v20.8h, v20.8b, #0 +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h +; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h +; CHECK-NEXT: mov v21.s[0], v7.s[0] +; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h +; CHECK-NEXT: sshll v2.8h, v2.8b, #0 +; CHECK-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h +; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h +; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h +; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h +; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h +; CHECK-NEXT: add v5.4s, v18.4s, v22.4s +; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h +; CHECK-NEXT: add v0.4s, v6.4s, v21.4s +; CHECK-NEXT: add v2.4s, v5.4s, v16.4s +; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v4.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret +entry: + %az = zext <25 x i8> %a to <25 x i32> + %bz = sext <25 x i8> %b to <25 x i32> + %m1 = mul nuw nsw <25 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1) + %cz = zext <25 x i8> %c to <25 x i32> + %dz = sext <25 x i8> %d to <25 x i32> + %m2 = mul nuw nsw <25 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-SD-LABEL: test_udot_v32i8: ; CHECK-SD: // %bb.0: // %entry @@ -2293,14 +3728,210 @@ define i32 @test_sdot_v32i8_double_nomla(<32 x i8> %a, <32 x i8> %b, <32 x i8> % ; CHECK-GI-NEXT: add w0, w8, w9 ; CHECK-GI-NEXT: ret entry: - %az = sext <32 x i8> %a to <32 x i32> - %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az) - %cz = sext <32 x i8> %c to <32 x i32> - %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz) + %az = sext <32 x i8> %a to <32 x i32> + %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %az) + %cz = sext <32 x i8> %c to <32 x i32> + %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %cz) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_usdot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_v32i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 +; CHECK-SD-NEXT: ldp q2, q3, [x0] +; CHECK-SD-NEXT: ldp q4, q5, [x1] +; CHECK-SD-NEXT: usdot v1.4s, v3.16b, v5.16b +; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v4.16b +; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v32i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldp q0, q1, [x1] +; CHECK-GI-NEXT: ldp q2, q3, [x0] +; CHECK-GI-NEXT: sshll v4.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v5.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: ushll v6.8h, v2.8b, #0 +; CHECK-GI-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-GI-NEXT: ushll v7.8h, v3.8b, #0 +; CHECK-GI-NEXT: ushll2 v3.8h, v3.16b, #0 +; CHECK-GI-NEXT: sshll2 v16.4s, v4.8h, #0 +; CHECK-GI-NEXT: sshll2 v17.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll2 v18.4s, v5.8h, #0 +; CHECK-GI-NEXT: sshll2 v19.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll2 v20.4s, v6.8h, #0 +; CHECK-GI-NEXT: ushll2 v21.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll2 v22.4s, v7.8h, #0 +; CHECK-GI-NEXT: ushll2 v23.4s, v3.8h, #0 +; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0 +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v5.4s, v5.4h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: mul v16.4s, v16.4s, v20.4s +; CHECK-GI-NEXT: mul v17.4s, v17.4s, v21.4s +; CHECK-GI-NEXT: ushll v6.4s, v6.4h, #0 +; CHECK-GI-NEXT: mul v18.4s, v18.4s, v22.4s +; CHECK-GI-NEXT: mul v19.4s, v19.4s, v23.4s +; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll v7.4s, v7.4h, #0 +; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: mla v16.4s, v4.4s, v6.4s +; CHECK-GI-NEXT: mla v17.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: mla v18.4s, v5.4s, v7.4s +; CHECK-GI-NEXT: mla v19.4s, v1.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v16.4s, v17.4s +; CHECK-GI-NEXT: add v1.4s, v18.4s, v19.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ret +entry: + %0 = load <32 x i8>, ptr %a + %1 = zext <32 x i8> %0 to <32 x i32> + %2 = load <32 x i8>, ptr %b + %3 = sext <32 x i8> %2 to <32 x i32> + %4 = mul nsw <32 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v32i8_double(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v32i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v16.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v17.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v18.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v19.2d, #0000000000000000 +; CHECK-SD-NEXT: usdot v16.4s, v1.16b, v3.16b +; CHECK-SD-NEXT: usdot v18.4s, v0.16b, v2.16b +; CHECK-SD-NEXT: usdot v17.4s, v4.16b, v6.16b +; CHECK-SD-NEXT: usdot v19.4s, v5.16b, v7.16b +; CHECK-SD-NEXT: add v0.4s, v18.4s, v16.4s +; CHECK-SD-NEXT: add v1.4s, v17.4s, v19.4s +; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v32i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 64 +; CHECK-GI-NEXT: .cfi_offset b8, -8 +; CHECK-GI-NEXT: .cfi_offset b9, -16 +; CHECK-GI-NEXT: .cfi_offset b10, -24 +; CHECK-GI-NEXT: .cfi_offset b11, -32 +; CHECK-GI-NEXT: .cfi_offset b12, -40 +; CHECK-GI-NEXT: .cfi_offset b13, -48 +; CHECK-GI-NEXT: .cfi_offset b14, -56 +; CHECK-GI-NEXT: .cfi_offset b15, -64 +; CHECK-GI-NEXT: ushll v16.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ushll v17.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: sshll v18.8h, v2.8b, #0 +; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-GI-NEXT: sshll v19.8h, v3.8b, #0 +; CHECK-GI-NEXT: sshll2 v3.8h, v3.16b, #0 +; CHECK-GI-NEXT: ushll v27.8h, v4.8b, #0 +; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0 +; CHECK-GI-NEXT: ushll v28.8h, v5.8b, #0 +; CHECK-GI-NEXT: sshll v29.8h, v6.8b, #0 +; CHECK-GI-NEXT: sshll2 v6.8h, v6.16b, #0 +; CHECK-GI-NEXT: ushll2 v5.8h, v5.16b, #0 +; CHECK-GI-NEXT: sshll v30.8h, v7.8b, #0 +; CHECK-GI-NEXT: sshll2 v7.8h, v7.16b, #0 +; CHECK-GI-NEXT: ushll2 v20.4s, v16.8h, #0 +; CHECK-GI-NEXT: ushll2 v21.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll2 v22.4s, v17.8h, #0 +; CHECK-GI-NEXT: ushll2 v23.4s, v1.8h, #0 +; CHECK-GI-NEXT: sshll2 v24.4s, v18.8h, #0 +; CHECK-GI-NEXT: sshll2 v25.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll2 v26.4s, v19.8h, #0 +; CHECK-GI-NEXT: sshll2 v31.4s, v3.8h, #0 +; CHECK-GI-NEXT: ushll2 v8.4s, v27.8h, #0 +; CHECK-GI-NEXT: ushll2 v9.4s, v4.8h, #0 +; CHECK-GI-NEXT: ushll2 v10.4s, v28.8h, #0 +; CHECK-GI-NEXT: sshll2 v11.4s, v29.8h, #0 +; CHECK-GI-NEXT: sshll2 v12.4s, v6.8h, #0 +; CHECK-GI-NEXT: ushll2 v13.4s, v5.8h, #0 +; CHECK-GI-NEXT: sshll2 v14.4s, v30.8h, #0 +; CHECK-GI-NEXT: sshll2 v15.4s, v7.8h, #0 +; CHECK-GI-NEXT: mul v20.4s, v20.4s, v24.4s +; CHECK-GI-NEXT: mul v21.4s, v21.4s, v25.4s +; CHECK-GI-NEXT: mul v22.4s, v22.4s, v26.4s +; CHECK-GI-NEXT: mul v23.4s, v23.4s, v31.4s +; CHECK-GI-NEXT: mul v24.4s, v8.4s, v11.4s +; CHECK-GI-NEXT: mul v25.4s, v9.4s, v12.4s +; CHECK-GI-NEXT: ushll v16.4s, v16.4h, #0 +; CHECK-GI-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: mul v26.4s, v10.4s, v14.4s +; CHECK-GI-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: mul v31.4s, v13.4s, v15.4s +; CHECK-GI-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll v18.4s, v18.4h, #0 +; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll v19.4s, v19.4h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll v27.4s, v27.4h, #0 +; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0 +; CHECK-GI-NEXT: ushll v28.4s, v28.4h, #0 +; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0 +; CHECK-GI-NEXT: sshll v29.4s, v29.4h, #0 +; CHECK-GI-NEXT: sshll v6.4s, v6.4h, #0 +; CHECK-GI-NEXT: sshll v30.4s, v30.4h, #0 +; CHECK-GI-NEXT: sshll v7.4s, v7.4h, #0 +; CHECK-GI-NEXT: mla v20.4s, v16.4s, v18.4s +; CHECK-GI-NEXT: mla v21.4s, v0.4s, v2.4s +; CHECK-GI-NEXT: mla v22.4s, v17.4s, v19.4s +; CHECK-GI-NEXT: mla v23.4s, v1.4s, v3.4s +; CHECK-GI-NEXT: mla v24.4s, v27.4s, v29.4s +; CHECK-GI-NEXT: mla v25.4s, v4.4s, v6.4s +; CHECK-GI-NEXT: mla v26.4s, v28.4s, v30.4s +; CHECK-GI-NEXT: mla v31.4s, v5.4s, v7.4s +; CHECK-GI-NEXT: add v0.4s, v20.4s, v21.4s +; CHECK-GI-NEXT: add v1.4s, v22.4s, v23.4s +; CHECK-GI-NEXT: add v2.4s, v24.4s, v25.4s +; CHECK-GI-NEXT: add v3.4s, v26.4s, v31.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload +; CHECK-GI-NEXT: ret +entry: + %az = zext <32 x i8> %a to <32 x i32> + %bz = sext <32 x i8> %b to <32 x i32> + %m1 = mul nuw nsw <32 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m1) + %cz = zext <32 x i8> %c to <32 x i32> + %dz = sext <32 x i8> %d to <32 x i32> + %m2 = mul nuw nsw <32 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %m2) %x = add i32 %r1, %r2 ret i32 %x } + define i32 @test_udot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-LABEL: test_udot_v33i8: ; CHECK: // %bb.0: // %entry @@ -2866,6 +4497,362 @@ entry: %x = add i32 %r1, %r2 ret i32 %x } + +define i32 @test_usdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_usdot_v33i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldr b0, [x0, #32] +; CHECK-NEXT: ldr b1, [x1, #32] +; CHECK-NEXT: ldp q2, q4, [x0] +; CHECK-NEXT: ldp q3, q6, [x1] +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: ushll v5.8h, v2.8b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: ushll v16.8h, v4.8b, #0 +; CHECK-NEXT: sshll v7.8h, v3.8b, #0 +; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0 +; CHECK-NEXT: ushll2 v4.8h, v4.16b, #0 +; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h +; CHECK-NEXT: movi v1.2d, #0000000000000000 +; CHECK-NEXT: sshll v19.8h, v6.8b, #0 +; CHECK-NEXT: sshll2 v6.8h, v6.16b, #0 +; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h +; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h +; CHECK-NEXT: mov v1.s[0], v0.s[0] +; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h +; CHECK-NEXT: smlal2 v18.4s, v6.8h, v4.8h +; CHECK-NEXT: smlal2 v17.4s, v19.8h, v16.8h +; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h +; CHECK-NEXT: smlal v0.4s, v6.4h, v4.4h +; CHECK-NEXT: add v2.4s, v17.4s, v18.4s +; CHECK-NEXT: smlal v1.4s, v19.4h, v16.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret +entry: + %0 = load <33 x i8>, ptr %a + %1 = zext <33 x i8> %0 to <33 x i32> + %2 = load <33 x i8>, ptr %b + %3 = sext <33 x i8> %2 to <33 x i32> + %4 = mul nsw <33 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) { +; CHECK-LABEL: test_usdot_v33i8_double: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: fmov s4, w0 +; CHECK-NEXT: ldr b0, [sp, #144] +; CHECK-NEXT: add x8, sp, #152 +; CHECK-NEXT: ldr b1, [sp, #80] +; CHECK-NEXT: add x10, sp, #88 +; CHECK-NEXT: ldr b6, [sp, #16] +; CHECK-NEXT: ld1 { v0.b }[1], [x8] +; CHECK-NEXT: ldr b2, [sp, #408] +; CHECK-NEXT: add x9, sp, #160 +; CHECK-NEXT: mov v4.b[1], w1 +; CHECK-NEXT: ld1 { v1.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #24 +; CHECK-NEXT: ld1 { v6.b }[1], [x10] +; CHECK-NEXT: add x10, sp, #416 +; CHECK-NEXT: add x8, sp, #168 +; CHECK-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #96 +; CHECK-NEXT: ld1 { v2.b }[1], [x10] +; CHECK-NEXT: ld1 { v1.b }[2], [x9] +; CHECK-NEXT: add x9, sp, #32 +; CHECK-NEXT: add x10, sp, #424 +; CHECK-NEXT: mov v4.b[2], w2 +; CHECK-NEXT: ld1 { v6.b }[2], [x9] +; CHECK-NEXT: add x12, sp, #176 +; CHECK-NEXT: ld1 { v2.b }[2], [x10] +; CHECK-NEXT: add x10, sp, #104 +; CHECK-NEXT: ld1 { v0.b }[3], [x8] +; CHECK-NEXT: ld1 { v1.b }[3], [x10] +; CHECK-NEXT: add x10, sp, #40 +; CHECK-NEXT: add x13, sp, #112 +; CHECK-NEXT: ld1 { v6.b }[3], [x10] +; CHECK-NEXT: add x10, sp, #432 +; CHECK-NEXT: add x11, sp, #184 +; CHECK-NEXT: mov v4.b[3], w3 +; CHECK-NEXT: ld1 { v0.b }[4], [x12] +; CHECK-NEXT: add x12, sp, #48 +; CHECK-NEXT: ld1 { v2.b }[3], [x10] +; CHECK-NEXT: ld1 { v1.b }[4], [x13] +; CHECK-NEXT: add x14, sp, #440 +; CHECK-NEXT: ld1 { v6.b }[4], [x12] +; CHECK-NEXT: ldr b5, [sp, #216] +; CHECK-NEXT: add x10, sp, #120 +; CHECK-NEXT: ld1 { v0.b }[5], [x11] +; CHECK-NEXT: add x11, sp, #56 +; CHECK-NEXT: ldr b7, [sp, #280] +; CHECK-NEXT: mov v4.b[4], w4 +; CHECK-NEXT: ld1 { v2.b }[4], [x14] +; CHECK-NEXT: add x14, sp, #224 +; CHECK-NEXT: ld1 { v6.b }[5], [x11] +; CHECK-NEXT: ld1 { v5.b }[1], [x14] +; CHECK-NEXT: add x15, sp, #288 +; CHECK-NEXT: ld1 { v1.b }[5], [x10] +; CHECK-NEXT: add x14, sp, #64 +; CHECK-NEXT: ld1 { v7.b }[1], [x15] +; CHECK-NEXT: add x9, sp, #192 +; CHECK-NEXT: add x12, sp, #128 +; CHECK-NEXT: ldr b3, [sp, #344] +; CHECK-NEXT: mov v4.b[5], w5 +; CHECK-NEXT: ld1 { v6.b }[6], [x14] +; CHECK-NEXT: add x14, sp, #232 +; CHECK-NEXT: ld1 { v0.b }[6], [x9] +; CHECK-NEXT: ld1 { v5.b }[2], [x14] +; CHECK-NEXT: ld1 { v1.b }[6], [x12] +; CHECK-NEXT: add x12, sp, #296 +; CHECK-NEXT: add x13, sp, #352 +; CHECK-NEXT: add x8, sp, #200 +; CHECK-NEXT: ld1 { v7.b }[2], [x12] +; CHECK-NEXT: ld1 { v3.b }[1], [x13] +; CHECK-NEXT: add x14, sp, #240 +; CHECK-NEXT: mov v4.b[6], w6 +; CHECK-NEXT: add x10, sp, #448 +; CHECK-NEXT: ld1 { v5.b }[3], [x14] +; CHECK-NEXT: ld1 { v0.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #304 +; CHECK-NEXT: add x9, sp, #360 +; CHECK-NEXT: ld1 { v2.b }[5], [x10] +; CHECK-NEXT: ld1 { v7.b }[3], [x8] +; CHECK-NEXT: add x11, sp, #72 +; CHECK-NEXT: ld1 { v3.b }[2], [x9] +; CHECK-NEXT: add x8, sp, #248 +; CHECK-NEXT: add x15, sp, #456 +; CHECK-NEXT: mov v4.b[7], w7 +; CHECK-NEXT: ld1 { v6.b }[7], [x11] +; CHECK-NEXT: ld1 { v5.b }[4], [x8] +; CHECK-NEXT: add x8, sp, #312 +; CHECK-NEXT: ld1 { v2.b }[6], [x15] +; CHECK-NEXT: add x9, sp, #368 +; CHECK-NEXT: ld1 { v7.b }[4], [x8] +; CHECK-NEXT: ld1 { v3.b }[3], [x9] +; CHECK-NEXT: add x8, sp, #256 +; CHECK-NEXT: ushll v17.8h, v6.8b, #0 +; CHECK-NEXT: add x10, sp, #464 +; CHECK-NEXT: ld1 { v5.b }[5], [x8] +; CHECK-NEXT: ushll v21.8h, v4.8b, #0 +; CHECK-NEXT: ldr b4, [sp, #208] +; CHECK-NEXT: add x8, sp, #320 +; CHECK-NEXT: ld1 { v2.b }[7], [x10] +; CHECK-NEXT: add x9, sp, #376 +; CHECK-NEXT: ld1 { v7.b }[5], [x8] +; CHECK-NEXT: ushll v6.8h, v4.8b, #0 +; CHECK-NEXT: ldr b4, [sp, #672] +; CHECK-NEXT: add x10, sp, #680 +; CHECK-NEXT: ld1 { v3.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #264 +; CHECK-NEXT: ldr b16, [sp, #472] +; CHECK-NEXT: ld1 { v4.b }[1], [x10] +; CHECK-NEXT: ld1 { v5.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #328 +; CHECK-NEXT: ld1 { v7.b }[6], [x9] +; CHECK-NEXT: add x10, sp, #688 +; CHECK-NEXT: add x9, sp, #272 +; CHECK-NEXT: add x8, sp, #384 +; CHECK-NEXT: sshll v16.8h, v16.8b, #0 +; CHECK-NEXT: ldr b19, [sp, #480] +; CHECK-NEXT: ld1 { v4.b }[2], [x10] +; CHECK-NEXT: ld1 { v5.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #336 +; CHECK-NEXT: ld1 { v3.b }[5], [x8] +; CHECK-NEXT: ld1 { v7.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #696 +; CHECK-NEXT: add x8, sp, #392 +; CHECK-NEXT: smull v20.4s, v6.4h, v16.4h +; CHECK-NEXT: ldr b16, [sp, #608] +; CHECK-NEXT: ld1 { v4.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #488 +; CHECK-NEXT: sshll v22.8h, v5.8b, #0 +; CHECK-NEXT: ld1 { v19.b }[1], [x9] +; CHECK-NEXT: ld1 { v3.b }[6], [x8] +; CHECK-NEXT: add x9, sp, #704 +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: add x10, sp, #496 +; CHECK-NEXT: add x8, sp, #400 +; CHECK-NEXT: ld1 { v4.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #616 +; CHECK-NEXT: sshll v18.8h, v7.8b, #0 +; CHECK-NEXT: ld1 { v16.b }[1], [x9] +; CHECK-NEXT: ld1 { v19.b }[2], [x10] +; CHECK-NEXT: ld1 { v3.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #712 +; CHECK-NEXT: add x9, sp, #504 +; CHECK-NEXT: smull2 v7.4s, v21.8h, v22.8h +; CHECK-NEXT: ld1 { v4.b }[5], [x8] +; CHECK-NEXT: add x8, sp, #624 +; CHECK-NEXT: mov v5.s[0], v20.s[0] +; CHECK-NEXT: ld1 { v16.b }[2], [x8] +; CHECK-NEXT: ld1 { v19.b }[3], [x9] +; CHECK-NEXT: add x8, sp, #720 +; CHECK-NEXT: add x9, sp, #512 +; CHECK-NEXT: ldr b20, [sp, #544] +; CHECK-NEXT: add x10, sp, #640 +; CHECK-NEXT: ld1 { v4.b }[6], [x8] +; CHECK-NEXT: add x8, sp, #632 +; CHECK-NEXT: add x11, sp, #520 +; CHECK-NEXT: ld1 { v16.b }[3], [x8] +; CHECK-NEXT: ld1 { v19.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #552 +; CHECK-NEXT: smlal v5.4s, v21.4h, v22.4h +; CHECK-NEXT: ld1 { v20.b }[1], [x9] +; CHECK-NEXT: ldr b21, [sp, #736] +; CHECK-NEXT: ldr b22, [sp, #1000] +; CHECK-NEXT: add x9, sp, #528 +; CHECK-NEXT: ldr b23, [sp, #808] +; CHECK-NEXT: ld1 { v16.b }[4], [x10] +; CHECK-NEXT: ld1 { v19.b }[5], [x11] +; CHECK-NEXT: add x10, sp, #560 +; CHECK-NEXT: ushll v21.8h, v21.8b, #0 +; CHECK-NEXT: sshll v24.8h, v22.8b, #0 +; CHECK-NEXT: ld1 { v20.b }[2], [x10] +; CHECK-NEXT: smull v6.4s, v17.4h, v18.4h +; CHECK-NEXT: smull2 v18.4s, v17.8h, v18.8h +; CHECK-NEXT: movi v17.2d, #0000000000000000 +; CHECK-NEXT: ld1 { v19.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #568 +; CHECK-NEXT: ldr b22, [sp, #744] +; CHECK-NEXT: add x11, sp, #816 +; CHECK-NEXT: smull v24.4s, v21.4h, v24.4h +; CHECK-NEXT: ld1 { v20.b }[3], [x9] +; CHECK-NEXT: add x10, sp, #752 +; CHECK-NEXT: ld1 { v23.b }[1], [x11] +; CHECK-NEXT: add x8, sp, #728 +; CHECK-NEXT: ld1 { v22.b }[1], [x10] +; CHECK-NEXT: add x9, sp, #576 +; CHECK-NEXT: ldr b21, [sp, #936] +; CHECK-NEXT: add x10, sp, #824 +; CHECK-NEXT: ld1 { v4.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #648 +; CHECK-NEXT: ld1 { v20.b }[4], [x9] +; CHECK-NEXT: add x9, sp, #760 +; CHECK-NEXT: add x11, sp, #944 +; CHECK-NEXT: ld1 { v23.b }[2], [x10] +; CHECK-NEXT: mov v17.s[0], v24.s[0] +; CHECK-NEXT: ldr b24, [sp, #872] +; CHECK-NEXT: ld1 { v16.b }[5], [x8] +; CHECK-NEXT: add x8, sp, #536 +; CHECK-NEXT: ld1 { v22.b }[2], [x9] +; CHECK-NEXT: ld1 { v21.b }[1], [x11] +; CHECK-NEXT: add x10, sp, #880 +; CHECK-NEXT: ld1 { v19.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #584 +; CHECK-NEXT: add x9, sp, #832 +; CHECK-NEXT: ld1 { v24.b }[1], [x10] +; CHECK-NEXT: ld1 { v20.b }[5], [x8] +; CHECK-NEXT: add x8, sp, #768 +; CHECK-NEXT: ld1 { v23.b }[3], [x9] +; CHECK-NEXT: add x9, sp, #952 +; CHECK-NEXT: ld1 { v22.b }[3], [x8] +; CHECK-NEXT: add x11, sp, #888 +; CHECK-NEXT: ld1 { v21.b }[2], [x9] +; CHECK-NEXT: add x8, sp, #592 +; CHECK-NEXT: add x10, sp, #840 +; CHECK-NEXT: ld1 { v24.b }[2], [x11] +; CHECK-NEXT: add x9, sp, #776 +; CHECK-NEXT: ld1 { v23.b }[4], [x10] +; CHECK-NEXT: ld1 { v20.b }[6], [x8] +; CHECK-NEXT: add x8, sp, #960 +; CHECK-NEXT: ld1 { v22.b }[4], [x9] +; CHECK-NEXT: ld1 { v21.b }[3], [x8] +; CHECK-NEXT: add x10, sp, #896 +; CHECK-NEXT: add x9, sp, #848 +; CHECK-NEXT: ld1 { v24.b }[3], [x10] +; CHECK-NEXT: add x8, sp, #784 +; CHECK-NEXT: ld1 { v23.b }[5], [x9] +; CHECK-NEXT: add x9, sp, #968 +; CHECK-NEXT: ld1 { v22.b }[5], [x8] +; CHECK-NEXT: add x11, sp, #904 +; CHECK-NEXT: ld1 { v21.b }[4], [x9] +; CHECK-NEXT: add x8, sp, #600 +; CHECK-NEXT: add x10, sp, #856 +; CHECK-NEXT: ld1 { v24.b }[4], [x11] +; CHECK-NEXT: add x9, sp, #792 +; CHECK-NEXT: ld1 { v23.b }[6], [x10] +; CHECK-NEXT: ld1 { v20.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #976 +; CHECK-NEXT: ld1 { v22.b }[6], [x9] +; CHECK-NEXT: ld1 { v21.b }[5], [x8] +; CHECK-NEXT: add x10, sp, #912 +; CHECK-NEXT: add x9, sp, #864 +; CHECK-NEXT: ld1 { v24.b }[5], [x10] +; CHECK-NEXT: add x8, sp, #800 +; CHECK-NEXT: ld1 { v23.b }[7], [x9] +; CHECK-NEXT: add x9, sp, #984 +; CHECK-NEXT: ld1 { v22.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #656 +; CHECK-NEXT: ld1 { v21.b }[6], [x9] +; CHECK-NEXT: add x9, sp, #920 +; CHECK-NEXT: ld1 { v16.b }[6], [x8] +; CHECK-NEXT: ld1 { v24.b }[6], [x9] +; CHECK-NEXT: add x8, sp, #992 +; CHECK-NEXT: add x13, sp, #136 +; CHECK-NEXT: ushll v19.8h, v19.8b, #0 +; CHECK-NEXT: ushll v20.8h, v20.8b, #0 +; CHECK-NEXT: sshll v22.8h, v22.8b, #0 +; CHECK-NEXT: sshll v23.8h, v23.8b, #0 +; CHECK-NEXT: add x9, sp, #664 +; CHECK-NEXT: ld1 { v21.b }[7], [x8] +; CHECK-NEXT: add x8, sp, #928 +; CHECK-NEXT: ld1 { v1.b }[7], [x13] +; CHECK-NEXT: ld1 { v16.b }[7], [x9] +; CHECK-NEXT: ld1 { v24.b }[7], [x8] +; CHECK-NEXT: smlal v17.4s, v19.4h, v22.4h +; CHECK-NEXT: smull2 v19.4s, v19.8h, v22.8h +; CHECK-NEXT: smull v22.4s, v20.4h, v23.4h +; CHECK-NEXT: smull2 v20.4s, v20.8h, v23.8h +; CHECK-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-NEXT: sshll v2.8h, v2.8b, #0 +; CHECK-NEXT: ushll v4.8h, v4.8b, #0 +; CHECK-NEXT: sshll v21.8h, v21.8b, #0 +; CHECK-NEXT: ushll v16.8h, v16.8b, #0 +; CHECK-NEXT: sshll v23.8h, v24.8b, #0 +; CHECK-NEXT: smlal2 v7.4s, v1.8h, v3.8h +; CHECK-NEXT: smlal v5.4s, v1.4h, v3.4h +; CHECK-NEXT: smlal2 v18.4s, v0.8h, v2.8h +; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h +; CHECK-NEXT: smlal2 v20.4s, v4.8h, v21.8h +; CHECK-NEXT: smlal v22.4s, v4.4h, v21.4h +; CHECK-NEXT: smlal2 v19.4s, v16.8h, v23.8h +; CHECK-NEXT: smlal v17.4s, v16.4h, v23.4h +; CHECK-NEXT: add v0.4s, v7.4s, v18.4s +; CHECK-NEXT: add v1.4s, v5.4s, v6.4s +; CHECK-NEXT: add v2.4s, v19.4s, v20.4s +; CHECK-NEXT: add v3.4s, v17.4s, v22.4s +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: add v1.4s, v3.4s, v2.4s +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret +entry: + %az = zext <33 x i8> %a to <33 x i32> + %bz = sext <33 x i8> %b to <33 x i32> + %m1 = mul nuw nsw <33 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1) + %cz = zext <33 x i8> %c to <33 x i32> + %dz = sext <33 x i8> %d to <33 x i32> + %m2 = mul nuw nsw <33 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-SD-LABEL: test_udot_v48i8: ; CHECK-SD: // %bb.0: // %entry @@ -3243,1003 +5230,2082 @@ define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 ; CHECK-SD-NEXT: add x8, sp, #1120 ; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9] ; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #736 -; CHECK-SD-NEXT: ldr b7, [sp, #1232] -; CHECK-SD-NEXT: ldr b16, [sp, #848] -; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10] -; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8] -; CHECK-SD-NEXT: add x9, sp, #1240 -; CHECK-SD-NEXT: add x10, sp, #856 -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10] -; CHECK-SD-NEXT: add x8, sp, #1128 -; CHECK-SD-NEXT: add x11, sp, #744 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8] -; CHECK-SD-NEXT: add x10, sp, #1248 -; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11] -; CHECK-SD-NEXT: add x11, sp, #864 -; CHECK-SD-NEXT: add x9, sp, #144 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11] -; CHECK-SD-NEXT: add x8, sp, #1136 -; CHECK-SD-NEXT: add x12, sp, #752 -; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12] -; CHECK-SD-NEXT: add x9, sp, #1256 -; CHECK-SD-NEXT: add x10, sp, #872 -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10] -; CHECK-SD-NEXT: add x8, sp, #1144 -; CHECK-SD-NEXT: add x11, sp, #760 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8] -; CHECK-SD-NEXT: add x10, sp, #1264 -; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11] -; CHECK-SD-NEXT: add x11, sp, #880 -; CHECK-SD-NEXT: add x9, sp, #152 -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11] -; CHECK-SD-NEXT: add x8, sp, #1152 -; CHECK-SD-NEXT: add x12, sp, #768 -; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12] -; CHECK-SD-NEXT: add x9, sp, #1272 -; CHECK-SD-NEXT: add x10, sp, #888 -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10] -; CHECK-SD-NEXT: add x8, sp, #1160 -; CHECK-SD-NEXT: add x11, sp, #776 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8] -; CHECK-SD-NEXT: add x10, sp, #1280 -; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11] -; CHECK-SD-NEXT: add x11, sp, #896 -; CHECK-SD-NEXT: add x9, sp, #160 +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #736 +; CHECK-SD-NEXT: ldr b7, [sp, #1232] +; CHECK-SD-NEXT: ldr b16, [sp, #848] +; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10] +; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8] +; CHECK-SD-NEXT: add x9, sp, #1240 +; CHECK-SD-NEXT: add x10, sp, #856 +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10] +; CHECK-SD-NEXT: add x8, sp, #1128 +; CHECK-SD-NEXT: add x11, sp, #744 +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8] +; CHECK-SD-NEXT: add x10, sp, #1248 +; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11] +; CHECK-SD-NEXT: add x11, sp, #864 +; CHECK-SD-NEXT: add x9, sp, #144 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11] +; CHECK-SD-NEXT: add x8, sp, #1136 +; CHECK-SD-NEXT: add x12, sp, #752 +; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12] +; CHECK-SD-NEXT: add x9, sp, #1256 +; CHECK-SD-NEXT: add x10, sp, #872 +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10] +; CHECK-SD-NEXT: add x8, sp, #1144 +; CHECK-SD-NEXT: add x11, sp, #760 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8] +; CHECK-SD-NEXT: add x10, sp, #1264 +; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11] +; CHECK-SD-NEXT: add x11, sp, #880 +; CHECK-SD-NEXT: add x9, sp, #152 +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11] +; CHECK-SD-NEXT: add x8, sp, #1152 +; CHECK-SD-NEXT: add x12, sp, #768 +; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12] +; CHECK-SD-NEXT: add x9, sp, #1272 +; CHECK-SD-NEXT: add x10, sp, #888 +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-SD-NEXT: add x8, sp, #1160 +; CHECK-SD-NEXT: add x11, sp, #776 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8] +; CHECK-SD-NEXT: add x10, sp, #1280 +; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11] +; CHECK-SD-NEXT: add x11, sp, #896 +; CHECK-SD-NEXT: add x9, sp, #160 +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11] +; CHECK-SD-NEXT: add x8, sp, #1168 +; CHECK-SD-NEXT: add x12, sp, #784 +; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12] +; CHECK-SD-NEXT: add x9, sp, #1288 +; CHECK-SD-NEXT: add x10, sp, #904 +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10] +; CHECK-SD-NEXT: add x8, sp, #1176 +; CHECK-SD-NEXT: add x11, sp, #792 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8] +; CHECK-SD-NEXT: add x10, sp, #1296 +; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11] +; CHECK-SD-NEXT: add x11, sp, #912 +; CHECK-SD-NEXT: add x9, sp, #168 +; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11] +; CHECK-SD-NEXT: add x8, sp, #1184 +; CHECK-SD-NEXT: add x12, sp, #800 +; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12] +; CHECK-SD-NEXT: add x9, sp, #1304 +; CHECK-SD-NEXT: add x10, sp, #920 +; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10] +; CHECK-SD-NEXT: add x8, sp, #1192 +; CHECK-SD-NEXT: add x11, sp, #808 +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8] +; CHECK-SD-NEXT: add x10, sp, #1312 +; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11] +; CHECK-SD-NEXT: add x11, sp, #928 +; CHECK-SD-NEXT: add x9, sp, #176 +; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11] +; CHECK-SD-NEXT: add x8, sp, #1200 +; CHECK-SD-NEXT: add x12, sp, #816 +; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12] +; CHECK-SD-NEXT: add x9, sp, #1320 +; CHECK-SD-NEXT: add x10, sp, #936 +; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10] +; CHECK-SD-NEXT: add x8, sp, #1208 +; CHECK-SD-NEXT: add x11, sp, #824 +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8] +; CHECK-SD-NEXT: add x10, sp, #1328 +; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11] +; CHECK-SD-NEXT: add x11, sp, #944 +; CHECK-SD-NEXT: add x9, sp, #184 +; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11] +; CHECK-SD-NEXT: add x8, sp, #1216 +; CHECK-SD-NEXT: add x12, sp, #832 +; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12] +; CHECK-SD-NEXT: add x9, sp, #1336 +; CHECK-SD-NEXT: add x10, sp, #952 +; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10] +; CHECK-SD-NEXT: add x8, sp, #1224 +; CHECK-SD-NEXT: add x11, sp, #840 +; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] +; CHECK-SD-NEXT: add x8, sp, #192 +; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11] +; CHECK-SD-NEXT: add x10, sp, #1344 +; CHECK-SD-NEXT: add x11, sp, #960 +; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11] +; CHECK-SD-NEXT: add x9, sp, #584 +; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b +; CHECK-SD-NEXT: add x8, sp, #200 +; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b +; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #1352 +; CHECK-SD-NEXT: add x10, sp, #968 +; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10] +; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b +; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b +; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_sdot_v48i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 +; CHECK-GI-NEXT: .cfi_offset w29, -16 +; CHECK-GI-NEXT: ldr w11, [sp, #80] +; CHECK-GI-NEXT: ldr w10, [sp, #208] +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: ldr w8, [sp, #88] +; CHECK-GI-NEXT: ldr w12, [sp, #344] +; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 +; CHECK-GI-NEXT: fmov s1, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #336] +; CHECK-GI-NEXT: fmov s2, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #464] +; CHECK-GI-NEXT: ldr w9, [sp, #216] +; CHECK-GI-NEXT: mov v0.b[1], w1 +; CHECK-GI-NEXT: fmov s3, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #600] +; CHECK-GI-NEXT: movi v21.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v1.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #592] +; CHECK-GI-NEXT: fmov s4, w10 +; CHECK-GI-NEXT: mov v2.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #472] +; CHECK-GI-NEXT: ldr w10, [sp, #608] +; CHECK-GI-NEXT: mov v3.b[1], w12 +; CHECK-GI-NEXT: fmov s5, w8 +; CHECK-GI-NEXT: ldr w8, [sp, #96] +; CHECK-GI-NEXT: mov v4.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #224] +; CHECK-GI-NEXT: mov v0.b[2], w2 +; CHECK-GI-NEXT: mov v1.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #352] +; CHECK-GI-NEXT: ldr w12, [sp, #848] +; CHECK-GI-NEXT: mov v2.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #480] +; CHECK-GI-NEXT: mov v5.b[1], w11 +; CHECK-GI-NEXT: mov v3.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #104] +; CHECK-GI-NEXT: ldr w11, [sp, #16] +; CHECK-GI-NEXT: mov v4.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #232] +; CHECK-GI-NEXT: mov v0.b[3], w3 +; CHECK-GI-NEXT: mov v1.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #360] +; CHECK-GI-NEXT: fmov s7, w12 +; CHECK-GI-NEXT: mov v2.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #488] +; CHECK-GI-NEXT: mov v5.b[2], w10 +; CHECK-GI-NEXT: mov v3.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #112] +; CHECK-GI-NEXT: ldr w10, [sp, #616] +; CHECK-GI-NEXT: mov v4.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #240] +; CHECK-GI-NEXT: mov v0.b[4], w4 +; CHECK-GI-NEXT: mov v1.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #368] +; CHECK-GI-NEXT: ldr w12, [sp, #1112] +; CHECK-GI-NEXT: mov v2.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #496] +; CHECK-GI-NEXT: mov v5.b[3], w10 +; CHECK-GI-NEXT: mov v3.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #120] +; CHECK-GI-NEXT: ldr w10, [sp, #624] +; CHECK-GI-NEXT: mov v4.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #248] +; CHECK-GI-NEXT: mov v0.b[5], w5 +; CHECK-GI-NEXT: mov v1.b[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #376] +; CHECK-GI-NEXT: movi v22.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #504] +; CHECK-GI-NEXT: mov v5.b[4], w10 +; CHECK-GI-NEXT: mov v3.b[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #128] +; CHECK-GI-NEXT: ldr w10, [sp, #632] +; CHECK-GI-NEXT: mov v4.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #256] +; CHECK-GI-NEXT: mov v0.b[6], w6 +; CHECK-GI-NEXT: mov v1.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #384] +; CHECK-GI-NEXT: movi v23.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #512] +; CHECK-GI-NEXT: mov v5.b[5], w10 +; CHECK-GI-NEXT: mov v3.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #136] +; CHECK-GI-NEXT: ldr w10, [sp, #640] +; CHECK-GI-NEXT: mov v4.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #264] +; CHECK-GI-NEXT: mov v0.b[7], w7 +; CHECK-GI-NEXT: mov v1.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #392] +; CHECK-GI-NEXT: movi v24.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #520] +; CHECK-GI-NEXT: mov v5.b[6], w10 +; CHECK-GI-NEXT: mov v3.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #144] +; CHECK-GI-NEXT: ldr w10, [sp, #648] +; CHECK-GI-NEXT: mov v4.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #272] +; CHECK-GI-NEXT: mov v0.b[8], w11 +; CHECK-GI-NEXT: mov v1.b[8], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #400] +; CHECK-GI-NEXT: ldr w11, [sp, #24] +; CHECK-GI-NEXT: mov v2.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #528] +; CHECK-GI-NEXT: mov v5.b[7], w10 +; CHECK-GI-NEXT: mov v3.b[8], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #152] +; CHECK-GI-NEXT: ldr w10, [sp, #656] +; CHECK-GI-NEXT: mov v4.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #280] +; CHECK-GI-NEXT: mov v0.b[9], w11 +; CHECK-GI-NEXT: mov v1.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #408] +; CHECK-GI-NEXT: ldr w11, [sp, #32] +; CHECK-GI-NEXT: mov v2.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #536] +; CHECK-GI-NEXT: mov v5.b[8], w10 +; CHECK-GI-NEXT: mov v3.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #160] +; CHECK-GI-NEXT: ldr w10, [sp, #664] +; CHECK-GI-NEXT: mov v4.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #288] +; CHECK-GI-NEXT: mov v0.b[10], w11 +; CHECK-GI-NEXT: mov v1.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #416] +; CHECK-GI-NEXT: ldr w11, [sp, #40] +; CHECK-GI-NEXT: mov v2.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #544] +; CHECK-GI-NEXT: mov v5.b[9], w10 +; CHECK-GI-NEXT: mov v3.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #168] +; CHECK-GI-NEXT: ldr w10, [sp, #672] +; CHECK-GI-NEXT: mov v4.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #296] +; CHECK-GI-NEXT: mov v0.b[11], w11 +; CHECK-GI-NEXT: mov v1.b[11], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #424] +; CHECK-GI-NEXT: ldr w11, [sp, #48] +; CHECK-GI-NEXT: mov v2.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #552] +; CHECK-GI-NEXT: mov v5.b[10], w10 +; CHECK-GI-NEXT: mov v3.b[11], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #176] +; CHECK-GI-NEXT: ldr w10, [sp, #680] +; CHECK-GI-NEXT: mov v4.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #304] +; CHECK-GI-NEXT: mov v0.b[12], w11 +; CHECK-GI-NEXT: mov v1.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #432] +; CHECK-GI-NEXT: ldr w11, [sp, #56] +; CHECK-GI-NEXT: mov v2.b[12], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #560] +; CHECK-GI-NEXT: mov v5.b[11], w10 +; CHECK-GI-NEXT: mov v3.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #184] +; CHECK-GI-NEXT: ldr w10, [sp, #688] +; CHECK-GI-NEXT: mov v4.b[12], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #312] +; CHECK-GI-NEXT: mov v0.b[13], w11 +; CHECK-GI-NEXT: mov v1.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #440] +; CHECK-GI-NEXT: ldr w11, [sp, #64] +; CHECK-GI-NEXT: mov v2.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #568] +; CHECK-GI-NEXT: mov v5.b[12], w10 +; CHECK-GI-NEXT: mov v3.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #192] +; CHECK-GI-NEXT: ldr w10, [sp, #696] +; CHECK-GI-NEXT: mov v4.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #320] +; CHECK-GI-NEXT: mov v0.b[14], w11 +; CHECK-GI-NEXT: mov v1.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #448] +; CHECK-GI-NEXT: ldr w11, [sp, #72] +; CHECK-GI-NEXT: mov v2.b[14], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #576] +; CHECK-GI-NEXT: mov v5.b[13], w10 +; CHECK-GI-NEXT: mov v3.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #720] +; CHECK-GI-NEXT: ldr w10, [sp, #704] +; CHECK-GI-NEXT: mov v4.b[14], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #728] +; CHECK-GI-NEXT: mov v0.b[15], w11 +; CHECK-GI-NEXT: fmov s6, w8 +; CHECK-GI-NEXT: ldr w8, [sp, #328] +; CHECK-GI-NEXT: ldr w11, [sp, #456] +; CHECK-GI-NEXT: mov v5.b[14], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #200] +; CHECK-GI-NEXT: movi v25.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[15], w8 +; CHECK-GI-NEXT: mov v3.b[15], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #736] +; CHECK-GI-NEXT: mov v6.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #584] +; CHECK-GI-NEXT: ldr w8, [sp, #856] +; CHECK-GI-NEXT: mov v1.b[15], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #712] +; CHECK-GI-NEXT: mov v4.b[15], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #976] +; CHECK-GI-NEXT: mov v7.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1232] +; CHECK-GI-NEXT: mov v5.b[15], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #984] +; CHECK-GI-NEXT: mov v6.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1104] +; CHECK-GI-NEXT: fmov s16, w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1360] +; CHECK-GI-NEXT: fmov s18, w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1368] +; CHECK-GI-NEXT: fmov s17, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1240] +; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b +; CHECK-GI-NEXT: mov v16.b[1], w10 +; CHECK-GI-NEXT: fmov s19, w9 +; CHECK-GI-NEXT: ldr w10, [sp, #864] +; CHECK-GI-NEXT: mov v18.b[1], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #992] +; CHECK-GI-NEXT: ldr w9, [sp, #1120] +; CHECK-GI-NEXT: mov v17.b[1], w12 +; CHECK-GI-NEXT: mov v7.b[2], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1248] +; CHECK-GI-NEXT: mov v19.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #744] +; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b +; CHECK-GI-NEXT: mov v16.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #872] +; CHECK-GI-NEXT: addv s0, v20.4s +; CHECK-GI-NEXT: mov v6.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1000] +; CHECK-GI-NEXT: mov v18.b[2], w10 +; CHECK-GI-NEXT: mov v17.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1376] +; CHECK-GI-NEXT: ldr w10, [sp, #1128] +; CHECK-GI-NEXT: mov v7.b[3], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #880] +; CHECK-GI-NEXT: addv s1, v21.4s +; CHECK-GI-NEXT: mov v19.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #752] +; CHECK-GI-NEXT: mov v16.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1256] +; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b +; CHECK-GI-NEXT: mov v17.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1384] +; CHECK-GI-NEXT: mov v6.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1008] +; CHECK-GI-NEXT: mov v18.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1136] +; CHECK-GI-NEXT: mov v19.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #760] +; CHECK-GI-NEXT: mov v7.b[4], w11 +; CHECK-GI-NEXT: mov v16.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1264] +; CHECK-GI-NEXT: ldr w11, [sp, #888] +; CHECK-GI-NEXT: mov v17.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1392] +; CHECK-GI-NEXT: mov v6.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1016] +; CHECK-GI-NEXT: mov v18.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1144] +; CHECK-GI-NEXT: mov v19.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #768] +; CHECK-GI-NEXT: mov v7.b[5], w11 +; CHECK-GI-NEXT: mov v16.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1272] +; CHECK-GI-NEXT: ldr w11, [sp, #896] +; CHECK-GI-NEXT: mov v17.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1400] +; CHECK-GI-NEXT: mov v6.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1024] +; CHECK-GI-NEXT: mov v18.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1152] +; CHECK-GI-NEXT: mov v19.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #776] +; CHECK-GI-NEXT: mov v7.b[6], w11 +; CHECK-GI-NEXT: mov v16.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1280] +; CHECK-GI-NEXT: ldr w11, [sp, #904] +; CHECK-GI-NEXT: mov v17.b[6], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1408] +; CHECK-GI-NEXT: mov v6.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1032] +; CHECK-GI-NEXT: mov v18.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1160] +; CHECK-GI-NEXT: mov v19.b[6], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #784] +; CHECK-GI-NEXT: mov v7.b[7], w11 +; CHECK-GI-NEXT: mov v16.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1288] +; CHECK-GI-NEXT: ldr w11, [sp, #912] +; CHECK-GI-NEXT: mov v17.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1416] +; CHECK-GI-NEXT: mov v6.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1040] +; CHECK-GI-NEXT: mov v18.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1168] +; CHECK-GI-NEXT: mov v19.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #792] +; CHECK-GI-NEXT: mov v7.b[8], w11 +; CHECK-GI-NEXT: mov v16.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1296] +; CHECK-GI-NEXT: ldr w11, [sp, #920] +; CHECK-GI-NEXT: mov v17.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1424] +; CHECK-GI-NEXT: mov v6.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1048] +; CHECK-GI-NEXT: mov v18.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1176] +; CHECK-GI-NEXT: mov v19.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #800] +; CHECK-GI-NEXT: mov v7.b[9], w11 +; CHECK-GI-NEXT: mov v16.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1304] +; CHECK-GI-NEXT: ldr w11, [sp, #928] +; CHECK-GI-NEXT: mov v17.b[9], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1432] +; CHECK-GI-NEXT: mov v6.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1056] +; CHECK-GI-NEXT: mov v18.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1184] +; CHECK-GI-NEXT: mov v19.b[9], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #808] +; CHECK-GI-NEXT: mov v7.b[10], w11 +; CHECK-GI-NEXT: mov v16.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1312] +; CHECK-GI-NEXT: ldr w11, [sp, #936] +; CHECK-GI-NEXT: mov v17.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1440] +; CHECK-GI-NEXT: mov v6.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1064] +; CHECK-GI-NEXT: mov v18.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1192] +; CHECK-GI-NEXT: mov v19.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #816] +; CHECK-GI-NEXT: mov v7.b[11], w11 +; CHECK-GI-NEXT: mov v16.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1320] +; CHECK-GI-NEXT: ldr w11, [sp, #944] +; CHECK-GI-NEXT: mov v17.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1448] +; CHECK-GI-NEXT: mov v6.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1072] +; CHECK-GI-NEXT: mov v18.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1200] +; CHECK-GI-NEXT: mov v19.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #824] +; CHECK-GI-NEXT: mov v7.b[12], w11 +; CHECK-GI-NEXT: mov v16.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1328] +; CHECK-GI-NEXT: ldr w11, [sp, #952] +; CHECK-GI-NEXT: mov v17.b[12], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1456] +; CHECK-GI-NEXT: mov v6.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1080] +; CHECK-GI-NEXT: mov v18.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1208] +; CHECK-GI-NEXT: mov v19.b[12], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #832] +; CHECK-GI-NEXT: mov v7.b[13], w11 +; CHECK-GI-NEXT: mov v16.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1336] +; CHECK-GI-NEXT: ldr w11, [sp, #960] +; CHECK-GI-NEXT: mov v17.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1464] +; CHECK-GI-NEXT: mov v6.b[14], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1088] +; CHECK-GI-NEXT: mov v18.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1216] +; CHECK-GI-NEXT: mov v19.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #840] +; CHECK-GI-NEXT: mov v7.b[14], w11 +; CHECK-GI-NEXT: mov v16.b[14], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1344] +; CHECK-GI-NEXT: ldr w11, [sp, #968] +; CHECK-GI-NEXT: mov v17.b[14], w9 +; CHECK-GI-NEXT: mov v6.b[15], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1096] +; CHECK-GI-NEXT: mov v18.b[14], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #1472] +; CHECK-GI-NEXT: ldr w10, [sp, #1224] +; CHECK-GI-NEXT: mov v7.b[15], w11 +; CHECK-GI-NEXT: addv s4, v25.4s +; CHECK-GI-NEXT: mov v16.b[15], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1352] +; CHECK-GI-NEXT: mov v19.b[14], w9 +; CHECK-GI-NEXT: mov v17.b[15], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #1480] +; CHECK-GI-NEXT: mov v18.b[15], w8 +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w11, s4 +; CHECK-GI-NEXT: mov v19.b[15], w9 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b +; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b +; CHECK-GI-NEXT: add w8, w8, w11 +; CHECK-GI-NEXT: addv s2, v22.4s +; CHECK-GI-NEXT: addv s3, v23.4s +; CHECK-GI-NEXT: addv s5, v24.4s +; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: add w9, w9, w10 +; CHECK-GI-NEXT: fmov w10, s5 +; CHECK-GI-NEXT: add w9, w9, w10 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-GI-NEXT: ret +entry: + %az = sext <48 x i8> %a to <48 x i32> + %bz = sext <48 x i8> %b to <48 x i32> + %m1 = mul nuw nsw <48 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1) + %cz = sext <48 x i8> %c to <48 x i32> + %dz = sext <48 x i8> %d to <48 x i32> + %m2 = mul nuw nsw <48 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { +; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 +; CHECK-SD-NEXT: .cfi_offset w29, -16 +; CHECK-SD-NEXT: ldr b5, [sp, #208] +; CHECK-SD-NEXT: add x8, sp, #216 +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: ldr b4, [sp, #976] +; CHECK-SD-NEXT: add x9, sp, #984 +; CHECK-SD-NEXT: add x12, sp, #328 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #224 +; CHECK-SD-NEXT: movi v1.16b, #1 +; CHECK-SD-NEXT: mov v0.b[1], w1 +; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9] +; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 +; CHECK-SD-NEXT: add x11, sp, #992 +; CHECK-SD-NEXT: ldr b6, [sp, #720] +; CHECK-SD-NEXT: ldr b7, [sp, #80] +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #232 +; CHECK-SD-NEXT: add x13, sp, #88 +; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11] +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13] +; CHECK-SD-NEXT: add x13, sp, #856 +; CHECK-SD-NEXT: mov v0.b[2], w2 +; CHECK-SD-NEXT: add x14, sp, #1008 +; CHECK-SD-NEXT: add x15, sp, #872 +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #240 +; CHECK-SD-NEXT: add x16, sp, #888 +; CHECK-SD-NEXT: add x10, sp, #16 +; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: add x11, sp, #40 +; CHECK-SD-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #248 +; CHECK-SD-NEXT: mov v0.b[3], w3 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #256 +; CHECK-SD-NEXT: mov v0.b[4], w4 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8] +; CHECK-SD-NEXT: add x8, sp, #264 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8] +; CHECK-SD-NEXT: add x8, sp, #272 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8] +; CHECK-SD-NEXT: add x8, sp, #280 +; CHECK-SD-NEXT: mov v0.b[6], w6 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8] +; CHECK-SD-NEXT: add x8, sp, #288 +; CHECK-SD-NEXT: mov v0.b[7], w7 +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #296 +; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10] +; CHECK-SD-NEXT: add x10, sp, #128 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #304 +; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9] +; CHECK-SD-NEXT: add x9, sp, #136 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #312 +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #320 +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 +; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #144 +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12] +; CHECK-SD-NEXT: add x12, sp, #728 +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] +; CHECK-SD-NEXT: add x12, sp, #1000 +; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11] +; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12] +; CHECK-SD-NEXT: add x12, sp, #736 +; CHECK-SD-NEXT: add x11, sp, #920 +; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b +; CHECK-SD-NEXT: ldr b5, [sp, #848] +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12] +; CHECK-SD-NEXT: add x12, sp, #48 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13] +; CHECK-SD-NEXT: add x13, sp, #744 +; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14] +; CHECK-SD-NEXT: add x14, sp, #96 +; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13] +; CHECK-SD-NEXT: add x13, sp, #864 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14] +; CHECK-SD-NEXT: add x14, sp, #1016 +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13] +; CHECK-SD-NEXT: add x13, sp, #752 +; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14] +; CHECK-SD-NEXT: add x14, sp, #104 +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13] +; CHECK-SD-NEXT: add x13, sp, #1024 +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15] +; CHECK-SD-NEXT: add x15, sp, #760 +; CHECK-SD-NEXT: add x14, sp, #112 +; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] +; CHECK-SD-NEXT: add x13, sp, #880 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15] +; CHECK-SD-NEXT: add x15, sp, #1032 +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13] +; CHECK-SD-NEXT: add x14, sp, #768 +; CHECK-SD-NEXT: add x13, sp, #120 +; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15] +; CHECK-SD-NEXT: add x15, sp, #1040 +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14] +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13] +; CHECK-SD-NEXT: add x13, sp, #776 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16] +; CHECK-SD-NEXT: add x14, sp, #1048 +; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15] +; CHECK-SD-NEXT: add x15, sp, #896 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13] ; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11] -; CHECK-SD-NEXT: add x8, sp, #1168 -; CHECK-SD-NEXT: add x12, sp, #784 -; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12] -; CHECK-SD-NEXT: add x9, sp, #1288 -; CHECK-SD-NEXT: add x10, sp, #904 +; CHECK-SD-NEXT: add x10, sp, #784 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15] +; CHECK-SD-NEXT: add x13, sp, #1056 +; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] +; CHECK-SD-NEXT: add x14, sp, #904 +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10] ; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10] -; CHECK-SD-NEXT: add x8, sp, #1176 -; CHECK-SD-NEXT: add x11, sp, #792 -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8] -; CHECK-SD-NEXT: add x10, sp, #1296 -; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11] -; CHECK-SD-NEXT: add x11, sp, #912 -; CHECK-SD-NEXT: add x9, sp, #168 -; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11] -; CHECK-SD-NEXT: add x8, sp, #1184 -; CHECK-SD-NEXT: add x12, sp, #800 -; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12] -; CHECK-SD-NEXT: add x9, sp, #1304 -; CHECK-SD-NEXT: add x10, sp, #920 -; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10] -; CHECK-SD-NEXT: add x8, sp, #1192 -; CHECK-SD-NEXT: add x11, sp, #808 -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8] -; CHECK-SD-NEXT: add x10, sp, #1312 -; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11] -; CHECK-SD-NEXT: add x11, sp, #928 -; CHECK-SD-NEXT: add x9, sp, #176 +; CHECK-SD-NEXT: add x9, sp, #792 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14] +; CHECK-SD-NEXT: add x10, sp, #1064 +; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13] +; CHECK-SD-NEXT: add x13, sp, #912 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8] +; CHECK-SD-NEXT: add x9, sp, #800 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13] +; CHECK-SD-NEXT: add x8, sp, #152 +; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10] +; CHECK-SD-NEXT: add x10, sp, #1072 +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8] +; CHECK-SD-NEXT: add x9, sp, #808 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11] +; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10] +; CHECK-SD-NEXT: add x10, sp, #160 +; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9] +; CHECK-SD-NEXT: add x9, sp, #928 ; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11] -; CHECK-SD-NEXT: add x8, sp, #1200 -; CHECK-SD-NEXT: add x12, sp, #816 -; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9] +; CHECK-SD-NEXT: add x10, sp, #1080 +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] +; CHECK-SD-NEXT: add x8, sp, #816 +; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10] +; CHECK-SD-NEXT: add x9, sp, #168 +; CHECK-SD-NEXT: add x10, sp, #176 ; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12] -; CHECK-SD-NEXT: add x9, sp, #1320 -; CHECK-SD-NEXT: add x10, sp, #936 +; CHECK-SD-NEXT: add x8, sp, #936 ; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10] -; CHECK-SD-NEXT: add x8, sp, #1208 -; CHECK-SD-NEXT: add x11, sp, #824 -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8] -; CHECK-SD-NEXT: add x10, sp, #1328 -; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11] -; CHECK-SD-NEXT: add x11, sp, #944 -; CHECK-SD-NEXT: add x9, sp, #184 +; CHECK-SD-NEXT: add x9, sp, #1088 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #64 +; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] +; CHECK-SD-NEXT: add x9, sp, #824 +; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] +; CHECK-SD-NEXT: add x9, sp, #944 ; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11] -; CHECK-SD-NEXT: add x8, sp, #1216 -; CHECK-SD-NEXT: add x12, sp, #832 -; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9] +; CHECK-SD-NEXT: add x10, sp, #1096 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9] +; CHECK-SD-NEXT: add x8, sp, #832 +; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10] +; CHECK-SD-NEXT: add x9, sp, #184 +; CHECK-SD-NEXT: add x10, sp, #72 ; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12] -; CHECK-SD-NEXT: add x9, sp, #1336 -; CHECK-SD-NEXT: add x10, sp, #952 +; CHECK-SD-NEXT: add x8, sp, #952 ; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10] -; CHECK-SD-NEXT: add x8, sp, #1224 -; CHECK-SD-NEXT: add x11, sp, #840 +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #840 +; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] +; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b +; CHECK-SD-NEXT: add x9, sp, #192 ; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #192 -; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11] -; CHECK-SD-NEXT: add x10, sp, #1344 -; CHECK-SD-NEXT: add x11, sp, #960 -; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11] -; CHECK-SD-NEXT: add x9, sp, #584 -; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b -; CHECK-SD-NEXT: add x8, sp, #200 -; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b -; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #1352 -; CHECK-SD-NEXT: add x10, sp, #968 -; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10] -; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b -; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b -; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s +; CHECK-SD-NEXT: add x8, sp, #960 +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] +; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b +; CHECK-SD-NEXT: add x8, sp, #200 +; CHECK-SD-NEXT: add x9, sp, #968 +; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] +; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b +; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b +; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s ; CHECK-SD-NEXT: addv s0, v0.4s ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-SD-NEXT: ret ; -; CHECK-GI-LABEL: test_sdot_v48i8_double: +; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 ; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: ldr w11, [sp, #80] -; CHECK-GI-NEXT: ldr w10, [sp, #208] +; CHECK-GI-NEXT: ldr w10, [sp, #80] +; CHECK-GI-NEXT: ldr w11, [sp, #208] ; CHECK-GI-NEXT: fmov s0, w0 -; CHECK-GI-NEXT: ldr w8, [sp, #88] -; CHECK-GI-NEXT: ldr w12, [sp, #344] -; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 -; CHECK-GI-NEXT: fmov s1, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #336] -; CHECK-GI-NEXT: fmov s2, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #464] -; CHECK-GI-NEXT: ldr w9, [sp, #216] +; CHECK-GI-NEXT: ldr w9, [sp, #88] +; CHECK-GI-NEXT: ldr w12, [sp, #728] +; CHECK-GI-NEXT: movi v6.16b, #1 +; CHECK-GI-NEXT: fmov s1, w10 +; CHECK-GI-NEXT: fmov s2, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #720] +; CHECK-GI-NEXT: ldr w10, [sp, #216] ; CHECK-GI-NEXT: mov v0.b[1], w1 +; CHECK-GI-NEXT: ldr w13, [sp, #856] ; CHECK-GI-NEXT: fmov s3, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #600] -; CHECK-GI-NEXT: movi v21.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v1.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #592] -; CHECK-GI-NEXT: fmov s4, w10 -; CHECK-GI-NEXT: mov v2.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #472] -; CHECK-GI-NEXT: ldr w10, [sp, #608] -; CHECK-GI-NEXT: mov v3.b[1], w12 -; CHECK-GI-NEXT: fmov s5, w8 ; CHECK-GI-NEXT: ldr w8, [sp, #96] -; CHECK-GI-NEXT: mov v4.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #224] +; CHECK-GI-NEXT: ldr w11, [sp, #224] +; CHECK-GI-NEXT: mov v1.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #848] +; CHECK-GI-NEXT: mov v2.b[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #976] +; CHECK-GI-NEXT: movi v7.2d, #0000000000000000 +; CHECK-GI-NEXT: movi v16.2d, #0000000000000000 +; CHECK-GI-NEXT: fmov s4, w9 +; CHECK-GI-NEXT: mov v3.b[1], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #984] +; CHECK-GI-NEXT: fmov s5, w10 ; CHECK-GI-NEXT: mov v0.b[2], w2 +; CHECK-GI-NEXT: ldr w10, [sp, #736] ; CHECK-GI-NEXT: mov v1.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #352] -; CHECK-GI-NEXT: ldr w12, [sp, #848] -; CHECK-GI-NEXT: mov v2.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #480] -; CHECK-GI-NEXT: mov v5.b[1], w11 -; CHECK-GI-NEXT: mov v3.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #104] -; CHECK-GI-NEXT: ldr w11, [sp, #16] -; CHECK-GI-NEXT: mov v4.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #232] +; CHECK-GI-NEXT: ldr w8, [sp, #864] +; CHECK-GI-NEXT: mov v2.b[2], w11 +; CHECK-GI-NEXT: mov v4.b[1], w13 +; CHECK-GI-NEXT: ldr w11, [sp, #992] +; CHECK-GI-NEXT: ldr w12, [sp, #776] +; CHECK-GI-NEXT: mov v5.b[1], w9 +; CHECK-GI-NEXT: mov v3.b[2], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #104] +; CHECK-GI-NEXT: ldr w10, [sp, #232] ; CHECK-GI-NEXT: mov v0.b[3], w3 -; CHECK-GI-NEXT: mov v1.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #360] -; CHECK-GI-NEXT: fmov s7, w12 -; CHECK-GI-NEXT: mov v2.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #488] -; CHECK-GI-NEXT: mov v5.b[2], w10 +; CHECK-GI-NEXT: movi v17.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v1.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #872] +; CHECK-GI-NEXT: movi v18.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v4.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #744] +; CHECK-GI-NEXT: mov v2.b[3], w10 +; CHECK-GI-NEXT: mov v5.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1000] +; CHECK-GI-NEXT: ldr w10, [sp, #240] ; CHECK-GI-NEXT: mov v3.b[3], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #112] -; CHECK-GI-NEXT: ldr w10, [sp, #616] -; CHECK-GI-NEXT: mov v4.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #240] ; CHECK-GI-NEXT: mov v0.b[4], w4 +; CHECK-GI-NEXT: movi v19.2d, #0000000000000000 +; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v4.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #752] ; CHECK-GI-NEXT: mov v1.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #368] -; CHECK-GI-NEXT: ldr w12, [sp, #1112] -; CHECK-GI-NEXT: mov v2.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #496] -; CHECK-GI-NEXT: mov v5.b[3], w10 -; CHECK-GI-NEXT: mov v3.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #120] -; CHECK-GI-NEXT: ldr w10, [sp, #624] -; CHECK-GI-NEXT: mov v4.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #248] +; CHECK-GI-NEXT: ldr w8, [sp, #880] +; CHECK-GI-NEXT: mov v5.b[3], w11 +; CHECK-GI-NEXT: mov v2.b[4], w10 +; CHECK-GI-NEXT: mov v3.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #120] +; CHECK-GI-NEXT: ldr w11, [sp, #1008] +; CHECK-GI-NEXT: ldr w10, [sp, #248] ; CHECK-GI-NEXT: mov v0.b[5], w5 -; CHECK-GI-NEXT: mov v1.b[5], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #376] -; CHECK-GI-NEXT: movi v22.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #504] -; CHECK-GI-NEXT: mov v5.b[4], w10 +; CHECK-GI-NEXT: mov v4.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #760] +; CHECK-GI-NEXT: mov v1.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #888] +; CHECK-GI-NEXT: mov v5.b[4], w11 +; CHECK-GI-NEXT: mov v2.b[5], w10 ; CHECK-GI-NEXT: mov v3.b[5], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #128] -; CHECK-GI-NEXT: ldr w10, [sp, #632] -; CHECK-GI-NEXT: mov v4.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #256] +; CHECK-GI-NEXT: ldr w11, [sp, #1016] +; CHECK-GI-NEXT: ldr w10, [sp, #256] ; CHECK-GI-NEXT: mov v0.b[6], w6 +; CHECK-GI-NEXT: mov v4.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #768] ; CHECK-GI-NEXT: mov v1.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #384] -; CHECK-GI-NEXT: movi v23.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #512] -; CHECK-GI-NEXT: mov v5.b[5], w10 -; CHECK-GI-NEXT: mov v3.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #136] -; CHECK-GI-NEXT: ldr w10, [sp, #640] -; CHECK-GI-NEXT: mov v4.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #264] +; CHECK-GI-NEXT: ldr w8, [sp, #896] +; CHECK-GI-NEXT: mov v5.b[5], w11 +; CHECK-GI-NEXT: mov v2.b[6], w10 +; CHECK-GI-NEXT: mov v3.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #136] +; CHECK-GI-NEXT: ldr w11, [sp, #1024] +; CHECK-GI-NEXT: ldr w10, [sp, #264] ; CHECK-GI-NEXT: mov v0.b[7], w7 -; CHECK-GI-NEXT: mov v1.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #392] -; CHECK-GI-NEXT: movi v24.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #520] -; CHECK-GI-NEXT: mov v5.b[6], w10 -; CHECK-GI-NEXT: mov v3.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #144] -; CHECK-GI-NEXT: ldr w10, [sp, #648] +; CHECK-GI-NEXT: mov v4.b[6], w8 +; CHECK-GI-NEXT: mov v1.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #904] +; CHECK-GI-NEXT: mov v5.b[6], w11 +; CHECK-GI-NEXT: mov v2.b[7], w10 +; CHECK-GI-NEXT: ldr w8, [sp, #16] +; CHECK-GI-NEXT: mov v3.b[7], w12 +; CHECK-GI-NEXT: ldr w10, [sp, #144] +; CHECK-GI-NEXT: ldr w12, [sp, #1032] +; CHECK-GI-NEXT: mov v0.b[8], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #784] +; CHECK-GI-NEXT: ldr w11, [sp, #272] ; CHECK-GI-NEXT: mov v4.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #272] -; CHECK-GI-NEXT: mov v0.b[8], w11 -; CHECK-GI-NEXT: mov v1.b[8], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #400] -; CHECK-GI-NEXT: ldr w11, [sp, #24] -; CHECK-GI-NEXT: mov v2.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #528] -; CHECK-GI-NEXT: mov v5.b[7], w10 +; CHECK-GI-NEXT: mov v1.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #912] +; CHECK-GI-NEXT: mov v5.b[7], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #24] +; CHECK-GI-NEXT: ldr w12, [sp, #1040] ; CHECK-GI-NEXT: mov v3.b[8], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #152] -; CHECK-GI-NEXT: ldr w10, [sp, #656] -; CHECK-GI-NEXT: mov v4.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #280] -; CHECK-GI-NEXT: mov v0.b[9], w11 +; CHECK-GI-NEXT: mov v2.b[8], w11 +; CHECK-GI-NEXT: mov v0.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #792] +; CHECK-GI-NEXT: ldr w11, [sp, #280] +; CHECK-GI-NEXT: mov v4.b[8], w10 ; CHECK-GI-NEXT: mov v1.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #408] -; CHECK-GI-NEXT: ldr w11, [sp, #32] -; CHECK-GI-NEXT: mov v2.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #536] -; CHECK-GI-NEXT: mov v5.b[8], w10 -; CHECK-GI-NEXT: mov v3.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #160] -; CHECK-GI-NEXT: ldr w10, [sp, #664] -; CHECK-GI-NEXT: mov v4.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #288] -; CHECK-GI-NEXT: mov v0.b[10], w11 -; CHECK-GI-NEXT: mov v1.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #416] -; CHECK-GI-NEXT: ldr w11, [sp, #40] -; CHECK-GI-NEXT: mov v2.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #544] -; CHECK-GI-NEXT: mov v5.b[9], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #920] +; CHECK-GI-NEXT: mov v5.b[8], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #32] +; CHECK-GI-NEXT: ldr w12, [sp, #1048] +; CHECK-GI-NEXT: mov v3.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #160] +; CHECK-GI-NEXT: mov v2.b[9], w11 +; CHECK-GI-NEXT: mov v0.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #800] +; CHECK-GI-NEXT: ldr w11, [sp, #288] +; CHECK-GI-NEXT: mov v4.b[9], w10 +; CHECK-GI-NEXT: mov v1.b[10], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #928] +; CHECK-GI-NEXT: mov v5.b[9], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #40] +; CHECK-GI-NEXT: ldr w12, [sp, #1056] ; CHECK-GI-NEXT: mov v3.b[10], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #168] -; CHECK-GI-NEXT: ldr w10, [sp, #672] -; CHECK-GI-NEXT: mov v4.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #296] -; CHECK-GI-NEXT: mov v0.b[11], w11 +; CHECK-GI-NEXT: mov v2.b[10], w11 +; CHECK-GI-NEXT: mov v0.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #808] +; CHECK-GI-NEXT: ldr w11, [sp, #296] +; CHECK-GI-NEXT: mov v4.b[10], w10 ; CHECK-GI-NEXT: mov v1.b[11], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #424] -; CHECK-GI-NEXT: ldr w11, [sp, #48] -; CHECK-GI-NEXT: mov v2.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #552] -; CHECK-GI-NEXT: mov v5.b[10], w10 -; CHECK-GI-NEXT: mov v3.b[11], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #176] -; CHECK-GI-NEXT: ldr w10, [sp, #680] -; CHECK-GI-NEXT: mov v4.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #304] -; CHECK-GI-NEXT: mov v0.b[12], w11 -; CHECK-GI-NEXT: mov v1.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #432] -; CHECK-GI-NEXT: ldr w11, [sp, #56] -; CHECK-GI-NEXT: mov v2.b[12], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #560] -; CHECK-GI-NEXT: mov v5.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #936] +; CHECK-GI-NEXT: mov v5.b[10], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #48] +; CHECK-GI-NEXT: ldr w12, [sp, #1064] +; CHECK-GI-NEXT: mov v3.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #176] +; CHECK-GI-NEXT: mov v2.b[11], w11 +; CHECK-GI-NEXT: mov v0.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #816] +; CHECK-GI-NEXT: ldr w11, [sp, #304] +; CHECK-GI-NEXT: mov v4.b[11], w10 +; CHECK-GI-NEXT: mov v1.b[12], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #944] +; CHECK-GI-NEXT: mov v5.b[11], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #56] +; CHECK-GI-NEXT: ldr w12, [sp, #1072] ; CHECK-GI-NEXT: mov v3.b[12], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #184] -; CHECK-GI-NEXT: ldr w10, [sp, #688] -; CHECK-GI-NEXT: mov v4.b[12], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #312] -; CHECK-GI-NEXT: mov v0.b[13], w11 -; CHECK-GI-NEXT: mov v1.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #440] -; CHECK-GI-NEXT: ldr w11, [sp, #64] -; CHECK-GI-NEXT: mov v2.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #568] -; CHECK-GI-NEXT: mov v5.b[12], w10 -; CHECK-GI-NEXT: mov v3.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #192] -; CHECK-GI-NEXT: ldr w10, [sp, #696] -; CHECK-GI-NEXT: mov v4.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #320] -; CHECK-GI-NEXT: mov v0.b[14], w11 -; CHECK-GI-NEXT: mov v1.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #448] -; CHECK-GI-NEXT: ldr w11, [sp, #72] -; CHECK-GI-NEXT: mov v2.b[14], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #576] -; CHECK-GI-NEXT: mov v5.b[13], w10 -; CHECK-GI-NEXT: mov v3.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #720] -; CHECK-GI-NEXT: ldr w10, [sp, #704] -; CHECK-GI-NEXT: mov v4.b[14], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #728] -; CHECK-GI-NEXT: mov v0.b[15], w11 -; CHECK-GI-NEXT: fmov s6, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #328] -; CHECK-GI-NEXT: ldr w11, [sp, #456] -; CHECK-GI-NEXT: mov v5.b[14], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #200] -; CHECK-GI-NEXT: movi v25.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[15], w8 -; CHECK-GI-NEXT: mov v3.b[15], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #736] -; CHECK-GI-NEXT: mov v6.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #584] -; CHECK-GI-NEXT: ldr w8, [sp, #856] -; CHECK-GI-NEXT: mov v1.b[15], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #712] -; CHECK-GI-NEXT: mov v4.b[15], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #976] -; CHECK-GI-NEXT: mov v7.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1232] -; CHECK-GI-NEXT: mov v5.b[15], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #984] -; CHECK-GI-NEXT: mov v6.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1104] -; CHECK-GI-NEXT: fmov s16, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1360] -; CHECK-GI-NEXT: fmov s18, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1368] -; CHECK-GI-NEXT: fmov s17, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1240] -; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b -; CHECK-GI-NEXT: mov v16.b[1], w10 -; CHECK-GI-NEXT: fmov s19, w9 -; CHECK-GI-NEXT: ldr w10, [sp, #864] -; CHECK-GI-NEXT: mov v18.b[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #992] -; CHECK-GI-NEXT: ldr w9, [sp, #1120] -; CHECK-GI-NEXT: mov v17.b[1], w12 -; CHECK-GI-NEXT: mov v7.b[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1248] -; CHECK-GI-NEXT: mov v19.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #744] -; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b -; CHECK-GI-NEXT: mov v16.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #872] -; CHECK-GI-NEXT: addv s0, v20.4s -; CHECK-GI-NEXT: mov v6.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1000] -; CHECK-GI-NEXT: mov v18.b[2], w10 -; CHECK-GI-NEXT: mov v17.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1376] -; CHECK-GI-NEXT: ldr w10, [sp, #1128] -; CHECK-GI-NEXT: mov v7.b[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #880] -; CHECK-GI-NEXT: addv s1, v21.4s -; CHECK-GI-NEXT: mov v19.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #752] -; CHECK-GI-NEXT: mov v16.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1256] -; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b -; CHECK-GI-NEXT: mov v17.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1384] -; CHECK-GI-NEXT: mov v6.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1008] -; CHECK-GI-NEXT: mov v18.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1136] -; CHECK-GI-NEXT: mov v19.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #760] -; CHECK-GI-NEXT: mov v7.b[4], w11 -; CHECK-GI-NEXT: mov v16.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1264] -; CHECK-GI-NEXT: ldr w11, [sp, #888] -; CHECK-GI-NEXT: mov v17.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1392] -; CHECK-GI-NEXT: mov v6.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1016] -; CHECK-GI-NEXT: mov v18.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1144] -; CHECK-GI-NEXT: mov v19.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #768] -; CHECK-GI-NEXT: mov v7.b[5], w11 -; CHECK-GI-NEXT: mov v16.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1272] -; CHECK-GI-NEXT: ldr w11, [sp, #896] -; CHECK-GI-NEXT: mov v17.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1400] -; CHECK-GI-NEXT: mov v6.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1024] -; CHECK-GI-NEXT: mov v18.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1152] -; CHECK-GI-NEXT: mov v19.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #776] -; CHECK-GI-NEXT: mov v7.b[6], w11 -; CHECK-GI-NEXT: mov v16.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1280] -; CHECK-GI-NEXT: ldr w11, [sp, #904] -; CHECK-GI-NEXT: mov v17.b[6], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1408] -; CHECK-GI-NEXT: mov v6.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1032] -; CHECK-GI-NEXT: mov v18.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1160] -; CHECK-GI-NEXT: mov v19.b[6], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #784] -; CHECK-GI-NEXT: mov v7.b[7], w11 -; CHECK-GI-NEXT: mov v16.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1288] -; CHECK-GI-NEXT: ldr w11, [sp, #912] -; CHECK-GI-NEXT: mov v17.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1416] -; CHECK-GI-NEXT: mov v6.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1040] -; CHECK-GI-NEXT: mov v18.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1168] -; CHECK-GI-NEXT: mov v19.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #792] -; CHECK-GI-NEXT: mov v7.b[8], w11 -; CHECK-GI-NEXT: mov v16.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1296] -; CHECK-GI-NEXT: ldr w11, [sp, #920] -; CHECK-GI-NEXT: mov v17.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1424] -; CHECK-GI-NEXT: mov v6.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1048] -; CHECK-GI-NEXT: mov v18.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1176] -; CHECK-GI-NEXT: mov v19.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #800] -; CHECK-GI-NEXT: mov v7.b[9], w11 -; CHECK-GI-NEXT: mov v16.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1304] -; CHECK-GI-NEXT: ldr w11, [sp, #928] -; CHECK-GI-NEXT: mov v17.b[9], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1432] -; CHECK-GI-NEXT: mov v6.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1056] -; CHECK-GI-NEXT: mov v18.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1184] -; CHECK-GI-NEXT: mov v19.b[9], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #808] -; CHECK-GI-NEXT: mov v7.b[10], w11 -; CHECK-GI-NEXT: mov v16.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1312] -; CHECK-GI-NEXT: ldr w11, [sp, #936] -; CHECK-GI-NEXT: mov v17.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1440] -; CHECK-GI-NEXT: mov v6.b[11], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1064] -; CHECK-GI-NEXT: mov v18.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1192] -; CHECK-GI-NEXT: mov v19.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #816] -; CHECK-GI-NEXT: mov v7.b[11], w11 -; CHECK-GI-NEXT: mov v16.b[11], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1320] -; CHECK-GI-NEXT: ldr w11, [sp, #944] -; CHECK-GI-NEXT: mov v17.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1448] -; CHECK-GI-NEXT: mov v6.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1072] -; CHECK-GI-NEXT: mov v18.b[11], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1200] -; CHECK-GI-NEXT: mov v19.b[11], w9 +; CHECK-GI-NEXT: mov v2.b[12], w11 +; CHECK-GI-NEXT: mov v0.b[13], w9 ; CHECK-GI-NEXT: ldr w9, [sp, #824] -; CHECK-GI-NEXT: mov v7.b[12], w11 -; CHECK-GI-NEXT: mov v16.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1328] -; CHECK-GI-NEXT: ldr w11, [sp, #952] -; CHECK-GI-NEXT: mov v17.b[12], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1456] -; CHECK-GI-NEXT: mov v6.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1080] -; CHECK-GI-NEXT: mov v18.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1208] -; CHECK-GI-NEXT: mov v19.b[12], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #832] -; CHECK-GI-NEXT: mov v7.b[13], w11 -; CHECK-GI-NEXT: mov v16.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1336] -; CHECK-GI-NEXT: ldr w11, [sp, #960] -; CHECK-GI-NEXT: mov v17.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1464] -; CHECK-GI-NEXT: mov v6.b[14], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1088] -; CHECK-GI-NEXT: mov v18.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1216] -; CHECK-GI-NEXT: mov v19.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #840] -; CHECK-GI-NEXT: mov v7.b[14], w11 -; CHECK-GI-NEXT: mov v16.b[14], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1344] -; CHECK-GI-NEXT: ldr w11, [sp, #968] -; CHECK-GI-NEXT: mov v17.b[14], w9 -; CHECK-GI-NEXT: mov v6.b[15], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1096] -; CHECK-GI-NEXT: mov v18.b[14], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #1472] -; CHECK-GI-NEXT: ldr w10, [sp, #1224] -; CHECK-GI-NEXT: mov v7.b[15], w11 -; CHECK-GI-NEXT: addv s4, v25.4s -; CHECK-GI-NEXT: mov v16.b[15], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1352] -; CHECK-GI-NEXT: mov v19.b[14], w9 -; CHECK-GI-NEXT: mov v17.b[15], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #1480] -; CHECK-GI-NEXT: mov v18.b[15], w8 +; CHECK-GI-NEXT: ldr w11, [sp, #312] +; CHECK-GI-NEXT: mov v4.b[12], w10 +; CHECK-GI-NEXT: mov v1.b[13], w8 +; CHECK-GI-NEXT: ldr w10, [sp, #952] +; CHECK-GI-NEXT: mov v5.b[12], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #64] +; CHECK-GI-NEXT: ldr w12, [sp, #1080] +; CHECK-GI-NEXT: mov v3.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #192] +; CHECK-GI-NEXT: mov v2.b[13], w11 +; CHECK-GI-NEXT: mov v0.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #832] +; CHECK-GI-NEXT: ldr w11, [sp, #320] +; CHECK-GI-NEXT: mov v4.b[13], w10 +; CHECK-GI-NEXT: mov v1.b[14], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #960] +; CHECK-GI-NEXT: mov v5.b[13], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #72] +; CHECK-GI-NEXT: ldr w12, [sp, #1088] +; CHECK-GI-NEXT: mov v3.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #200] +; CHECK-GI-NEXT: mov v2.b[14], w11 +; CHECK-GI-NEXT: mov v0.b[15], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #840] +; CHECK-GI-NEXT: ldr w11, [sp, #328] +; CHECK-GI-NEXT: mov v4.b[14], w10 +; CHECK-GI-NEXT: mov v1.b[15], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #968] +; CHECK-GI-NEXT: mov v5.b[14], w12 +; CHECK-GI-NEXT: ldr w10, [sp, #1096] +; CHECK-GI-NEXT: mov v3.b[15], w9 +; CHECK-GI-NEXT: mov v2.b[15], w11 +; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b +; CHECK-GI-NEXT: mov v4.b[15], w8 +; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b +; CHECK-GI-NEXT: mov v5.b[15], w10 +; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b +; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b +; CHECK-GI-NEXT: addv s0, v7.4s +; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b +; CHECK-GI-NEXT: addv s1, v16.4s +; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b +; CHECK-GI-NEXT: addv s2, v17.4s +; CHECK-GI-NEXT: addv s4, v20.4s ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w11, s4 -; CHECK-GI-NEXT: mov v19.b[15], w9 ; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b -; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b +; CHECK-GI-NEXT: addv s3, v18.4s +; CHECK-GI-NEXT: addv s5, v19.4s +; CHECK-GI-NEXT: fmov w10, s2 ; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b -; CHECK-GI-NEXT: add w8, w8, w11 -; CHECK-GI-NEXT: addv s2, v22.4s -; CHECK-GI-NEXT: addv s3, v23.4s -; CHECK-GI-NEXT: addv s5, v24.4s -; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: add w9, w9, w10 -; CHECK-GI-NEXT: fmov w10, s5 -; CHECK-GI-NEXT: add w9, w9, w10 +; CHECK-GI-NEXT: fmov w9, s4 +; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w10, w10, w11 +; CHECK-GI-NEXT: fmov w11, s5 +; CHECK-GI-NEXT: add w9, w10, w11 ; CHECK-GI-NEXT: add w0, w8, w9 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-GI-NEXT: ret entry: %az = sext <48 x i8> %a to <48 x i32> - %bz = sext <48 x i8> %b to <48 x i32> - %m1 = mul nuw nsw <48 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1) + %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az) %cz = sext <48 x i8> %c to <48 x i32> - %dz = sext <48 x i8> %d to <48 x i32> - %m2 = mul nuw nsw <48 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2) + %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz) %x = add i32 %r1, %r2 ret i32 %x } -define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { -; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla: +define i32 @test_usdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_v48i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q3, [x0, #32] +; CHECK-SD-NEXT: ldp q7, q2, [x0] +; CHECK-SD-NEXT: ldr q4, [x1] +; CHECK-SD-NEXT: ldr q5, [x1, #32] +; CHECK-SD-NEXT: movi v6.2d, #0000000000000000 +; CHECK-SD-NEXT: usdot v0.4s, v3.16b, v5.16b +; CHECK-SD-NEXT: usdot v1.4s, v7.16b, v4.16b +; CHECK-SD-NEXT: ldr q3, [x1, #16] +; CHECK-SD-NEXT: usdot v6.4s, v2.16b, v3.16b +; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-NEXT: add v0.4s, v0.4s, v6.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v48i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: ldp q0, q2, [x1] +; CHECK-GI-NEXT: ldr q7, [x1, #32] +; CHECK-GI-NEXT: ldp q1, q4, [x0] +; CHECK-GI-NEXT: ldr q6, [x0, #32] +; CHECK-GI-NEXT: sshll v20.8h, v7.8b, #0 +; CHECK-GI-NEXT: sshll v3.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v18.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v19.8h, v2.8b, #0 +; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v0.8h, v7.16b, #0 +; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 +; CHECK-GI-NEXT: ushll v7.8h, v4.8b, #0 +; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0 +; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0 +; CHECK-GI-NEXT: sshll v16.4s, v3.4h, #0 +; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: sshll v21.4s, v19.4h, #0 +; CHECK-GI-NEXT: ushll v17.4s, v5.4h, #0 +; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 +; CHECK-GI-NEXT: ushll v25.4s, v1.4h, #0 +; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0 +; CHECK-GI-NEXT: sshll v22.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: ushll v27.4s, v4.4h, #0 +; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0 +; CHECK-GI-NEXT: mul v3.4s, v3.4s, v5.4s +; CHECK-GI-NEXT: mul v16.4s, v16.4s, v17.4s +; CHECK-GI-NEXT: sshll v17.4s, v18.4h, #0 +; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0 +; CHECK-GI-NEXT: ushll v5.8h, v6.8b, #0 +; CHECK-GI-NEXT: ushll2 v6.8h, v6.16b, #0 +; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0 +; CHECK-GI-NEXT: sshll v23.4s, v20.4h, #0 +; CHECK-GI-NEXT: mul v2.4s, v2.4s, v4.4s +; CHECK-GI-NEXT: sshll v24.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0 +; CHECK-GI-NEXT: addv s3, v3.4s +; CHECK-GI-NEXT: addv s16, v16.4s +; CHECK-GI-NEXT: mul v1.4s, v18.4s, v1.4s +; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0 +; CHECK-GI-NEXT: ushll v18.4s, v6.4h, #0 +; CHECK-GI-NEXT: addv s2, v2.4s +; CHECK-GI-NEXT: fmov w9, s3 +; CHECK-GI-NEXT: ushll2 v3.4s, v7.8h, #0 +; CHECK-GI-NEXT: mul v7.4s, v17.4s, v25.4s +; CHECK-GI-NEXT: mul v17.4s, v21.4s, v26.4s +; CHECK-GI-NEXT: fmov w8, s16 +; CHECK-GI-NEXT: ushll v16.4s, v5.4h, #0 +; CHECK-GI-NEXT: mul v21.4s, v22.4s, v27.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 +; CHECK-GI-NEXT: mul v3.4s, v19.4s, v3.4s +; CHECK-GI-NEXT: mul v0.4s, v0.4s, v4.4s +; CHECK-GI-NEXT: fmov w12, s2 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: mul v6.4s, v23.4s, v16.4s +; CHECK-GI-NEXT: mul v16.4s, v24.4s, v18.4s +; CHECK-GI-NEXT: addv s17, v17.4s +; CHECK-GI-NEXT: fmov w10, s1 +; CHECK-GI-NEXT: mul v5.4s, v20.4s, v5.4s +; CHECK-GI-NEXT: addv s4, v21.4s +; CHECK-GI-NEXT: addv s7, v7.4s +; CHECK-GI-NEXT: addv s1, v3.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w11, s17 +; CHECK-GI-NEXT: addv s3, v6.4s +; CHECK-GI-NEXT: fmov w9, s7 +; CHECK-GI-NEXT: addv s2, v5.4s +; CHECK-GI-NEXT: fmov w13, s1 +; CHECK-GI-NEXT: add w10, w10, w11 +; CHECK-GI-NEXT: fmov w11, s4 +; CHECK-GI-NEXT: addv s4, v16.4s +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w9, w10, w13 +; CHECK-GI-NEXT: fmov w13, s0 +; CHECK-GI-NEXT: add w11, w11, w12 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: fmov w12, s3 +; CHECK-GI-NEXT: add w10, w11, w12 +; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: fmov w12, s4 +; CHECK-GI-NEXT: add w9, w10, w11 +; CHECK-GI-NEXT: add w10, w12, w13 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w9, w10, w2 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %0 = load <48 x i8>, ptr %a + %1 = zext <48 x i8> %0 to <48 x i32> + %2 = load <48 x i8>, ptr %b + %3 = sext <48 x i8> %2 to <48 x i32> + %4 = mul nsw <48 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v48i8_double: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 ; CHECK-SD-NEXT: .cfi_offset w29, -16 +; CHECK-SD-NEXT: ldr b2, [sp, #592] +; CHECK-SD-NEXT: add x8, sp, #600 ; CHECK-SD-NEXT: ldr b5, [sp, #208] +; CHECK-SD-NEXT: add x9, sp, #616 +; CHECK-SD-NEXT: ldr b0, [sp, #336] +; CHECK-SD-NEXT: add x10, sp, #624 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #216 -; CHECK-SD-NEXT: fmov s0, w0 -; CHECK-SD-NEXT: ldr b4, [sp, #976] -; CHECK-SD-NEXT: add x9, sp, #984 -; CHECK-SD-NEXT: add x12, sp, #328 +; CHECK-SD-NEXT: add x11, sp, #240 ; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #608 +; CHECK-SD-NEXT: ldr b4, [sp, #464] +; CHECK-SD-NEXT: ldr b6, [sp, #80] +; CHECK-SD-NEXT: ldr b7, [sp, #1360] +; CHECK-SD-NEXT: ldr b16, [sp, #976] +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #224 -; CHECK-SD-NEXT: movi v1.16b, #1 -; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9] -; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 -; CHECK-SD-NEXT: add x11, sp, #992 -; CHECK-SD-NEXT: ldr b6, [sp, #720] -; CHECK-SD-NEXT: ldr b7, [sp, #80] +; CHECK-SD-NEXT: fmov s1, w0 ; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #344 +; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #232 -; CHECK-SD-NEXT: add x13, sp, #88 -; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11] -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13] -; CHECK-SD-NEXT: add x13, sp, #856 -; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: add x14, sp, #1008 -; CHECK-SD-NEXT: add x15, sp, #872 +; CHECK-SD-NEXT: ldr b18, [sp, #720] +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #632 +; CHECK-SD-NEXT: mov v1.b[1], w1 ; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #240 -; CHECK-SD-NEXT: add x16, sp, #888 -; CHECK-SD-NEXT: add x10, sp, #16 -; CHECK-SD-NEXT: add x9, sp, #24 -; CHECK-SD-NEXT: add x11, sp, #40 -; CHECK-SD-NEXT: movi v2.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #248 -; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #256 -; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #264 -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8] -; CHECK-SD-NEXT: add x8, sp, #272 -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8] -; CHECK-SD-NEXT: add x8, sp, #280 -; CHECK-SD-NEXT: mov v0.b[6], w6 -; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8] -; CHECK-SD-NEXT: add x8, sp, #288 -; CHECK-SD-NEXT: mov v0.b[7], w7 -; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #296 -; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10] -; CHECK-SD-NEXT: add x10, sp, #128 -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #304 -; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9] -; CHECK-SD-NEXT: add x9, sp, #136 -; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #312 -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #320 -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] -; CHECK-SD-NEXT: add x8, sp, #32 -; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #144 -; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12] -; CHECK-SD-NEXT: add x12, sp, #728 -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] -; CHECK-SD-NEXT: add x12, sp, #1000 -; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11] -; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12] -; CHECK-SD-NEXT: add x12, sp, #736 -; CHECK-SD-NEXT: add x11, sp, #920 -; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b -; CHECK-SD-NEXT: ldr b5, [sp, #848] -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12] -; CHECK-SD-NEXT: add x12, sp, #48 -; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13] -; CHECK-SD-NEXT: add x13, sp, #744 -; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14] -; CHECK-SD-NEXT: add x14, sp, #96 -; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13] +; CHECK-SD-NEXT: add x8, sp, #472 +; CHECK-SD-NEXT: add x12, sp, #856 +; CHECK-SD-NEXT: ld1 { v4.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #640 ; CHECK-SD-NEXT: add x13, sp, #864 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14] -; CHECK-SD-NEXT: add x14, sp, #1016 -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13] -; CHECK-SD-NEXT: add x13, sp, #752 -; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14] -; CHECK-SD-NEXT: add x14, sp, #104 -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13] -; CHECK-SD-NEXT: add x13, sp, #1024 -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14] -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15] -; CHECK-SD-NEXT: add x15, sp, #760 -; CHECK-SD-NEXT: add x14, sp, #112 -; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] -; CHECK-SD-NEXT: add x13, sp, #880 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15] -; CHECK-SD-NEXT: add x15, sp, #1032 -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13] -; CHECK-SD-NEXT: add x14, sp, #768 -; CHECK-SD-NEXT: add x13, sp, #120 -; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15] -; CHECK-SD-NEXT: add x15, sp, #1040 -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14] -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13] -; CHECK-SD-NEXT: add x13, sp, #776 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16] -; CHECK-SD-NEXT: add x14, sp, #1048 -; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15] -; CHECK-SD-NEXT: add x15, sp, #896 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13] -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] -; CHECK-SD-NEXT: add x10, sp, #784 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15] -; CHECK-SD-NEXT: add x13, sp, #1056 -; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] -; CHECK-SD-NEXT: add x14, sp, #904 -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10] -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] -; CHECK-SD-NEXT: add x9, sp, #792 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14] -; CHECK-SD-NEXT: add x10, sp, #1064 -; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13] -; CHECK-SD-NEXT: add x13, sp, #912 -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] -; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8] -; CHECK-SD-NEXT: add x9, sp, #800 -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13] -; CHECK-SD-NEXT: add x8, sp, #152 -; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10] -; CHECK-SD-NEXT: add x10, sp, #1072 -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9] -; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8] -; CHECK-SD-NEXT: add x9, sp, #808 +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10] +; CHECK-SD-NEXT: add x10, sp, #88 +; CHECK-SD-NEXT: movi v17.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x10] +; CHECK-SD-NEXT: add x10, sp, #248 +; CHECK-SD-NEXT: add x11, sp, #272 +; CHECK-SD-NEXT: mov v1.b[2], w2 +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #1368 +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-SD-NEXT: add x9, sp, #984 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[1], [x9] +; CHECK-SD-NEXT: add x9, sp, #352 +; CHECK-SD-NEXT: add x10, sp, #648 +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] +; CHECK-SD-NEXT: ld1 { v0.b }[2], [x9] +; CHECK-SD-NEXT: add x9, sp, #256 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9] +; CHECK-SD-NEXT: add x8, sp, #480 +; CHECK-SD-NEXT: add x9, sp, #96 +; CHECK-SD-NEXT: ld1 { v4.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #656 +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x9] +; CHECK-SD-NEXT: ld1 { v2.b }[7], [x10] +; CHECK-SD-NEXT: add x9, sp, #264 +; CHECK-SD-NEXT: add x10, sp, #1376 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9] +; CHECK-SD-NEXT: add x9, sp, #664 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] +; CHECK-SD-NEXT: add x10, sp, #992 +; CHECK-SD-NEXT: mov v1.b[3], w3 +; CHECK-SD-NEXT: ld1 { v2.b }[8], [x8] +; CHECK-SD-NEXT: add x8, sp, #360 +; CHECK-SD-NEXT: ld1 { v16.b }[2], [x10] +; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8] +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11] +; CHECK-SD-NEXT: add x11, sp, #488 +; CHECK-SD-NEXT: add x10, sp, #672 +; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11] +; CHECK-SD-NEXT: add x11, sp, #280 +; CHECK-SD-NEXT: ld1 { v2.b }[9], [x9] +; CHECK-SD-NEXT: add x9, sp, #104 +; CHECK-SD-NEXT: add x8, sp, #680 +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #368 ; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11] -; CHECK-SD-NEXT: add x8, sp, #56 -; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10] +; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9] +; CHECK-SD-NEXT: add x9, sp, #1384 +; CHECK-SD-NEXT: add x11, sp, #688 +; CHECK-SD-NEXT: ld1 { v2.b }[10], [x10] +; CHECK-SD-NEXT: add x10, sp, #376 +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #288 +; CHECK-SD-NEXT: mov v1.b[4], w4 +; CHECK-SD-NEXT: ld1 { v0.b }[5], [x10] +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] +; CHECK-SD-NEXT: add x10, sp, #296 +; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #496 +; CHECK-SD-NEXT: add x9, sp, #1000 +; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #384 +; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9] +; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8] +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x10] +; CHECK-SD-NEXT: add x8, sp, #112 +; CHECK-SD-NEXT: ld1 { v2.b }[12], [x11] +; CHECK-SD-NEXT: add x11, sp, #392 +; CHECK-SD-NEXT: add x9, sp, #696 +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #304 +; CHECK-SD-NEXT: add x10, sp, #704 +; CHECK-SD-NEXT: ld1 { v0.b }[7], [x11] +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] +; CHECK-SD-NEXT: add x11, sp, #312 +; CHECK-SD-NEXT: ld1 { v2.b }[13], [x9] +; CHECK-SD-NEXT: add x9, sp, #504 +; CHECK-SD-NEXT: add x8, sp, #1392 +; CHECK-SD-NEXT: ld1 { v4.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #400 +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x8] +; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x11] +; CHECK-SD-NEXT: add x11, sp, #320 +; CHECK-SD-NEXT: ld1 { v2.b }[14], [x10] +; CHECK-SD-NEXT: add x10, sp, #120 +; CHECK-SD-NEXT: add x9, sp, #328 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10] +; CHECK-SD-NEXT: add x10, sp, #408 +; CHECK-SD-NEXT: add x8, sp, #712 +; CHECK-SD-NEXT: ld1 { v0.b }[9], [x10] +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x11] +; CHECK-SD-NEXT: add x11, sp, #416 +; CHECK-SD-NEXT: add x10, sp, #512 +; CHECK-SD-NEXT: ld1 { v2.b }[15], [x8] +; CHECK-SD-NEXT: add x8, sp, #128 +; CHECK-SD-NEXT: ld1 { v4.b }[6], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] +; CHECK-SD-NEXT: add x8, sp, #1016 +; CHECK-SD-NEXT: ld1 { v0.b }[10], [x11] +; CHECK-SD-NEXT: add x11, sp, #1400 +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #1008 +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x11] +; CHECK-SD-NEXT: add x11, sp, #1408 +; CHECK-SD-NEXT: ld1 { v16.b }[4], [x9] +; CHECK-SD-NEXT: add x9, sp, #520 +; CHECK-SD-NEXT: add x10, sp, #424 +; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9] +; CHECK-SD-NEXT: add x9, sp, #136 +; CHECK-SD-NEXT: ld1 { v0.b }[11], [x10] +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-SD-NEXT: add x10, sp, #1416 +; CHECK-SD-NEXT: ld1 { v16.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #528 +; CHECK-SD-NEXT: add x9, sp, #144 +; CHECK-SD-NEXT: ld1 { v4.b }[8], [x8] +; CHECK-SD-NEXT: add x8, sp, #1024 +; CHECK-SD-NEXT: add x11, sp, #1432 +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x9] +; CHECK-SD-NEXT: add x10, sp, #1424 +; CHECK-SD-NEXT: ld1 { v16.b }[6], [x8] +; CHECK-SD-NEXT: add x8, sp, #536 +; CHECK-SD-NEXT: add x9, sp, #152 +; CHECK-SD-NEXT: ld1 { v4.b }[9], [x8] +; CHECK-SD-NEXT: add x8, sp, #1032 +; CHECK-SD-NEXT: usdot v3.4s, v5.16b, v2.16b +; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] ; CHECK-SD-NEXT: add x10, sp, #160 +; CHECK-SD-NEXT: ld1 { v16.b }[7], [x8] +; CHECK-SD-NEXT: add x8, sp, #544 +; CHECK-SD-NEXT: add x9, sp, #432 +; CHECK-SD-NEXT: ld1 { v4.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #1040 +; CHECK-SD-NEXT: ld1 { v0.b }[12], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[9], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x10] +; CHECK-SD-NEXT: add x11, sp, #1440 +; CHECK-SD-NEXT: ld1 { v16.b }[8], [x8] +; CHECK-SD-NEXT: add x8, sp, #552 +; CHECK-SD-NEXT: add x10, sp, #168 +; CHECK-SD-NEXT: ld1 { v4.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #1048 +; CHECK-SD-NEXT: add x9, sp, #176 +; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x10] +; CHECK-SD-NEXT: add x10, sp, #1448 +; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8] +; CHECK-SD-NEXT: add x8, sp, #560 +; CHECK-SD-NEXT: add x11, sp, #576 +; CHECK-SD-NEXT: ld1 { v4.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #1056 +; CHECK-SD-NEXT: ldr b5, [sp, #1104] +; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9] +; CHECK-SD-NEXT: add x10, sp, #1456 +; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #568 +; CHECK-SD-NEXT: add x9, sp, #184 +; CHECK-SD-NEXT: ld1 { v4.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #1064 +; CHECK-SD-NEXT: mov v1.b[5], w5 +; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] +; CHECK-SD-NEXT: add x9, sp, #1072 +; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8] +; CHECK-SD-NEXT: add x10, sp, #192 +; CHECK-SD-NEXT: movi v2.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v4.b }[14], [x11] +; CHECK-SD-NEXT: add x11, sp, #1464 +; CHECK-SD-NEXT: add x8, sp, #440 +; CHECK-SD-NEXT: ld1 { v7.b }[13], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10] +; CHECK-SD-NEXT: add x11, sp, #1472 +; CHECK-SD-NEXT: ld1 { v16.b }[12], [x9] +; CHECK-SD-NEXT: add x9, sp, #584 +; CHECK-SD-NEXT: add x10, sp, #200 +; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #1080 +; CHECK-SD-NEXT: mov v1.b[6], w6 +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10] +; CHECK-SD-NEXT: add x10, sp, #1112 +; CHECK-SD-NEXT: ld1 { v16.b }[13], [x9] +; CHECK-SD-NEXT: add x11, sp, #1480 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10] +; CHECK-SD-NEXT: add x9, sp, #1088 +; CHECK-SD-NEXT: add x10, sp, #1120 ; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9] -; CHECK-SD-NEXT: add x9, sp, #928 -; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] -; CHECK-SD-NEXT: add x10, sp, #1080 +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x11] +; CHECK-SD-NEXT: add x11, sp, #728 +; CHECK-SD-NEXT: usdot v2.4s, v6.16b, v4.16b +; CHECK-SD-NEXT: ld1 { v18.b }[1], [x11] +; CHECK-SD-NEXT: ld1 { v16.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x10] +; CHECK-SD-NEXT: add x10, sp, #736 +; CHECK-SD-NEXT: add x9, sp, #1096 +; CHECK-SD-NEXT: ldr b4, [sp, #1232] +; CHECK-SD-NEXT: ldr b6, [sp, #848] +; CHECK-SD-NEXT: add x11, sp, #1240 +; CHECK-SD-NEXT: mov v1.b[7], w7 +; CHECK-SD-NEXT: ld1 { v18.b }[2], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #1128 +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x9] +; CHECK-SD-NEXT: add x10, sp, #744 +; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11] +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] +; CHECK-SD-NEXT: add x9, sp, #1136 +; CHECK-SD-NEXT: add x12, sp, #1248 +; CHECK-SD-NEXT: ld1 { v18.b }[3], [x10] +; CHECK-SD-NEXT: add x11, sp, #752 +; CHECK-SD-NEXT: add x10, sp, #16 +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[2], [x12] +; CHECK-SD-NEXT: add x9, sp, #1144 +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x13] +; CHECK-SD-NEXT: add x12, sp, #872 +; CHECK-SD-NEXT: ld1 { v1.b }[8], [x10] +; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11] +; CHECK-SD-NEXT: add x11, sp, #1256 +; CHECK-SD-NEXT: add x10, sp, #760 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11] +; CHECK-SD-NEXT: add x9, sp, #1152 +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x12] +; CHECK-SD-NEXT: add x12, sp, #1264 +; CHECK-SD-NEXT: add x13, sp, #880 +; CHECK-SD-NEXT: ld1 { v18.b }[5], [x10] +; CHECK-SD-NEXT: add x11, sp, #768 +; CHECK-SD-NEXT: add x10, sp, #24 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[4], [x12] +; CHECK-SD-NEXT: add x9, sp, #1160 +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13] +; CHECK-SD-NEXT: add x12, sp, #888 +; CHECK-SD-NEXT: ld1 { v1.b }[9], [x10] +; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11] +; CHECK-SD-NEXT: add x11, sp, #1272 +; CHECK-SD-NEXT: add x10, sp, #776 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[5], [x11] +; CHECK-SD-NEXT: add x9, sp, #1168 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x12] +; CHECK-SD-NEXT: add x12, sp, #1280 +; CHECK-SD-NEXT: add x13, sp, #896 +; CHECK-SD-NEXT: ld1 { v18.b }[7], [x10] +; CHECK-SD-NEXT: add x11, sp, #784 +; CHECK-SD-NEXT: add x10, sp, #32 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[6], [x12] +; CHECK-SD-NEXT: add x9, sp, #1176 +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x13] +; CHECK-SD-NEXT: add x12, sp, #904 +; CHECK-SD-NEXT: ld1 { v1.b }[10], [x10] +; CHECK-SD-NEXT: ld1 { v18.b }[8], [x11] +; CHECK-SD-NEXT: add x11, sp, #1288 +; CHECK-SD-NEXT: add x10, sp, #792 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[7], [x11] +; CHECK-SD-NEXT: add x9, sp, #1184 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x12] +; CHECK-SD-NEXT: add x12, sp, #1296 +; CHECK-SD-NEXT: add x13, sp, #912 +; CHECK-SD-NEXT: ld1 { v18.b }[9], [x10] +; CHECK-SD-NEXT: add x11, sp, #800 +; CHECK-SD-NEXT: add x10, sp, #40 ; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] -; CHECK-SD-NEXT: add x8, sp, #816 -; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10] -; CHECK-SD-NEXT: add x9, sp, #168 -; CHECK-SD-NEXT: add x10, sp, #176 -; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #936 -; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] -; CHECK-SD-NEXT: add x9, sp, #1088 -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #64 -; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] -; CHECK-SD-NEXT: add x9, sp, #824 -; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] -; CHECK-SD-NEXT: add x9, sp, #944 -; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] -; CHECK-SD-NEXT: add x10, sp, #1096 +; CHECK-SD-NEXT: ld1 { v4.b }[8], [x12] +; CHECK-SD-NEXT: add x9, sp, #1192 +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x13] +; CHECK-SD-NEXT: add x12, sp, #920 +; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10] +; CHECK-SD-NEXT: ld1 { v18.b }[10], [x11] +; CHECK-SD-NEXT: add x11, sp, #1304 +; CHECK-SD-NEXT: add x10, sp, #808 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[9], [x11] +; CHECK-SD-NEXT: add x9, sp, #1200 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12] +; CHECK-SD-NEXT: add x12, sp, #1312 +; CHECK-SD-NEXT: add x13, sp, #928 +; CHECK-SD-NEXT: ld1 { v18.b }[11], [x10] +; CHECK-SD-NEXT: add x11, sp, #816 +; CHECK-SD-NEXT: add x10, sp, #48 ; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9] -; CHECK-SD-NEXT: add x8, sp, #832 -; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10] -; CHECK-SD-NEXT: add x9, sp, #184 -; CHECK-SD-NEXT: add x10, sp, #72 -; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] -; CHECK-SD-NEXT: add x8, sp, #952 -; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #840 -; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] -; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b -; CHECK-SD-NEXT: add x9, sp, #192 -; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #960 -; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9] -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] -; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b -; CHECK-SD-NEXT: add x8, sp, #200 -; CHECK-SD-NEXT: add x9, sp, #968 -; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b -; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v4.b }[10], [x12] +; CHECK-SD-NEXT: add x9, sp, #1208 +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x13] +; CHECK-SD-NEXT: add x12, sp, #936 +; CHECK-SD-NEXT: ld1 { v1.b }[12], [x10] +; CHECK-SD-NEXT: ld1 { v18.b }[12], [x11] +; CHECK-SD-NEXT: add x11, sp, #1320 +; CHECK-SD-NEXT: add x10, sp, #824 +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[11], [x11] +; CHECK-SD-NEXT: add x9, sp, #1216 +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x12] +; CHECK-SD-NEXT: add x12, sp, #1328 +; CHECK-SD-NEXT: add x13, sp, #944 +; CHECK-SD-NEXT: ld1 { v18.b }[13], [x10] +; CHECK-SD-NEXT: add x11, sp, #832 +; CHECK-SD-NEXT: add x10, sp, #56 +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v4.b }[12], [x12] +; CHECK-SD-NEXT: add x9, sp, #1224 +; CHECK-SD-NEXT: ld1 { v6.b }[12], [x13] +; CHECK-SD-NEXT: ld1 { v1.b }[13], [x10] +; CHECK-SD-NEXT: add x10, sp, #1336 +; CHECK-SD-NEXT: ld1 { v18.b }[14], [x11] +; CHECK-SD-NEXT: add x11, sp, #952 +; CHECK-SD-NEXT: add x8, sp, #448 ; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] -; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b -; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b +; CHECK-SD-NEXT: add x9, sp, #840 +; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10] +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11] +; CHECK-SD-NEXT: usdot v17.4s, v16.16b, v7.16b +; CHECK-SD-NEXT: movi v7.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v18.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #64 +; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9] +; CHECK-SD-NEXT: add x9, sp, #1344 +; CHECK-SD-NEXT: add x10, sp, #960 +; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10] +; CHECK-SD-NEXT: add x8, sp, #456 +; CHECK-SD-NEXT: usdot v7.4s, v18.16b, v5.16b +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v0.b }[15], [x8] +; CHECK-SD-NEXT: add x8, sp, #72 +; CHECK-SD-NEXT: movi v16.2d, #0000000000000000 +; CHECK-SD-NEXT: add x9, sp, #1352 +; CHECK-SD-NEXT: add x10, sp, #968 +; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10] +; CHECK-SD-NEXT: usdot v16.4s, v1.16b, v0.16b ; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s +; CHECK-SD-NEXT: add v1.4s, v7.4s, v17.4s +; CHECK-SD-NEXT: usdot v5.4s, v6.16b, v4.16b +; CHECK-SD-NEXT: add v0.4s, v16.4s, v0.4s +; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s +; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-NEXT: addv s0, v0.4s ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-SD-NEXT: ret ; -; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla: +; CHECK-GI-LABEL: test_usdot_v48i8_double: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 ; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: ldr w10, [sp, #80] -; CHECK-GI-NEXT: ldr w11, [sp, #208] -; CHECK-GI-NEXT: fmov s0, w0 -; CHECK-GI-NEXT: ldr w9, [sp, #88] -; CHECK-GI-NEXT: ldr w12, [sp, #728] -; CHECK-GI-NEXT: movi v6.16b, #1 -; CHECK-GI-NEXT: fmov s1, w10 -; CHECK-GI-NEXT: fmov s2, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #720] -; CHECK-GI-NEXT: ldr w10, [sp, #216] -; CHECK-GI-NEXT: mov v0.b[1], w1 -; CHECK-GI-NEXT: ldr w13, [sp, #856] -; CHECK-GI-NEXT: fmov s3, w11 -; CHECK-GI-NEXT: ldr w8, [sp, #96] -; CHECK-GI-NEXT: ldr w11, [sp, #224] -; CHECK-GI-NEXT: mov v1.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #848] -; CHECK-GI-NEXT: mov v2.b[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #976] -; CHECK-GI-NEXT: movi v7.2d, #0000000000000000 -; CHECK-GI-NEXT: movi v16.2d, #0000000000000000 -; CHECK-GI-NEXT: fmov s4, w9 -; CHECK-GI-NEXT: mov v3.b[1], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #984] -; CHECK-GI-NEXT: fmov s5, w10 -; CHECK-GI-NEXT: mov v0.b[2], w2 -; CHECK-GI-NEXT: ldr w10, [sp, #736] -; CHECK-GI-NEXT: mov v1.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #864] -; CHECK-GI-NEXT: mov v2.b[2], w11 -; CHECK-GI-NEXT: mov v4.b[1], w13 -; CHECK-GI-NEXT: ldr w11, [sp, #992] -; CHECK-GI-NEXT: ldr w12, [sp, #776] -; CHECK-GI-NEXT: mov v5.b[1], w9 -; CHECK-GI-NEXT: mov v3.b[2], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #104] -; CHECK-GI-NEXT: ldr w10, [sp, #232] -; CHECK-GI-NEXT: mov v0.b[3], w3 -; CHECK-GI-NEXT: movi v17.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v1.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #872] -; CHECK-GI-NEXT: movi v18.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v4.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #744] -; CHECK-GI-NEXT: mov v2.b[3], w10 -; CHECK-GI-NEXT: mov v5.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1000] -; CHECK-GI-NEXT: ldr w10, [sp, #240] -; CHECK-GI-NEXT: mov v3.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #112] -; CHECK-GI-NEXT: mov v0.b[4], w4 -; CHECK-GI-NEXT: movi v19.2d, #0000000000000000 -; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v4.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #752] -; CHECK-GI-NEXT: mov v1.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #880] -; CHECK-GI-NEXT: mov v5.b[3], w11 -; CHECK-GI-NEXT: mov v2.b[4], w10 -; CHECK-GI-NEXT: mov v3.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #120] -; CHECK-GI-NEXT: ldr w11, [sp, #1008] -; CHECK-GI-NEXT: ldr w10, [sp, #248] -; CHECK-GI-NEXT: mov v0.b[5], w5 -; CHECK-GI-NEXT: mov v4.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #760] -; CHECK-GI-NEXT: mov v1.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #888] -; CHECK-GI-NEXT: mov v5.b[4], w11 -; CHECK-GI-NEXT: mov v2.b[5], w10 -; CHECK-GI-NEXT: mov v3.b[5], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #128] -; CHECK-GI-NEXT: ldr w11, [sp, #1016] -; CHECK-GI-NEXT: ldr w10, [sp, #256] -; CHECK-GI-NEXT: mov v0.b[6], w6 -; CHECK-GI-NEXT: mov v4.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #768] -; CHECK-GI-NEXT: mov v1.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #896] -; CHECK-GI-NEXT: mov v5.b[5], w11 -; CHECK-GI-NEXT: mov v2.b[6], w10 -; CHECK-GI-NEXT: mov v3.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #136] -; CHECK-GI-NEXT: ldr w11, [sp, #1024] -; CHECK-GI-NEXT: ldr w10, [sp, #264] -; CHECK-GI-NEXT: mov v0.b[7], w7 -; CHECK-GI-NEXT: mov v4.b[6], w8 -; CHECK-GI-NEXT: mov v1.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #904] -; CHECK-GI-NEXT: mov v5.b[6], w11 -; CHECK-GI-NEXT: mov v2.b[7], w10 ; CHECK-GI-NEXT: ldr w8, [sp, #16] -; CHECK-GI-NEXT: mov v3.b[7], w12 -; CHECK-GI-NEXT: ldr w10, [sp, #144] -; CHECK-GI-NEXT: ldr w12, [sp, #1032] -; CHECK-GI-NEXT: mov v0.b[8], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #784] -; CHECK-GI-NEXT: ldr w11, [sp, #272] -; CHECK-GI-NEXT: mov v4.b[7], w9 -; CHECK-GI-NEXT: mov v1.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #912] -; CHECK-GI-NEXT: mov v5.b[7], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #24] -; CHECK-GI-NEXT: ldr w12, [sp, #1040] -; CHECK-GI-NEXT: mov v3.b[8], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #152] -; CHECK-GI-NEXT: mov v2.b[8], w11 -; CHECK-GI-NEXT: mov v0.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #792] -; CHECK-GI-NEXT: ldr w11, [sp, #280] -; CHECK-GI-NEXT: mov v4.b[8], w10 -; CHECK-GI-NEXT: mov v1.b[9], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #920] -; CHECK-GI-NEXT: mov v5.b[8], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #80] +; CHECK-GI-NEXT: fmov s2, w0 +; CHECK-GI-NEXT: ldr w10, [sp, #24] +; CHECK-GI-NEXT: ldr w11, [sp, #144] +; CHECK-GI-NEXT: ldr w12, [sp, #296] +; CHECK-GI-NEXT: fmov s3, w8 +; CHECK-GI-NEXT: fmov s4, w9 +; CHECK-GI-NEXT: ldr w8, [sp, #88] +; CHECK-GI-NEXT: fmov s5, w11 +; CHECK-GI-NEXT: ldr w9, [sp, #152] +; CHECK-GI-NEXT: ldr w11, [sp, #216] +; CHECK-GI-NEXT: ldr w13, [sp, #64] +; CHECK-GI-NEXT: ldr w14, [sp, #128] +; CHECK-GI-NEXT: mov v2.b[1], w1 +; CHECK-GI-NEXT: mov v3.b[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #208] +; CHECK-GI-NEXT: mov v4.b[1], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #32] -; CHECK-GI-NEXT: ldr w12, [sp, #1048] -; CHECK-GI-NEXT: mov v3.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #160] -; CHECK-GI-NEXT: mov v2.b[9], w11 -; CHECK-GI-NEXT: mov v0.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #800] -; CHECK-GI-NEXT: ldr w11, [sp, #288] -; CHECK-GI-NEXT: mov v4.b[9], w10 -; CHECK-GI-NEXT: mov v1.b[10], w9 -; CHECK-GI-NEXT: ldr w10, [sp, #928] -; CHECK-GI-NEXT: mov v5.b[9], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #40] -; CHECK-GI-NEXT: ldr w12, [sp, #1056] -; CHECK-GI-NEXT: mov v3.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #168] -; CHECK-GI-NEXT: mov v2.b[10], w11 -; CHECK-GI-NEXT: mov v0.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #808] -; CHECK-GI-NEXT: ldr w11, [sp, #296] -; CHECK-GI-NEXT: mov v4.b[10], w10 -; CHECK-GI-NEXT: mov v1.b[11], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #936] -; CHECK-GI-NEXT: mov v5.b[10], w12 -; CHECK-GI-NEXT: ldr w8, [sp, #48] -; CHECK-GI-NEXT: ldr w12, [sp, #1064] -; CHECK-GI-NEXT: mov v3.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #176] -; CHECK-GI-NEXT: mov v2.b[11], w11 -; CHECK-GI-NEXT: mov v0.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #816] -; CHECK-GI-NEXT: ldr w11, [sp, #304] -; CHECK-GI-NEXT: mov v4.b[11], w10 -; CHECK-GI-NEXT: mov v1.b[12], w9 -; CHECK-GI-NEXT: ldr w10, [sp, #944] -; CHECK-GI-NEXT: mov v5.b[11], w12 +; CHECK-GI-NEXT: mov v5.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #96] +; CHECK-GI-NEXT: fmov s6, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #224] +; CHECK-GI-NEXT: ldr w15, [sp, #264] +; CHECK-GI-NEXT: ldr w16, [sp, #528] +; CHECK-GI-NEXT: mov v2.b[2], w2 +; CHECK-GI-NEXT: ldr w17, [sp, #648] +; CHECK-GI-NEXT: mov v3.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #160] +; CHECK-GI-NEXT: mov v4.b[2], w9 +; CHECK-GI-NEXT: mov v6.b[1], w11 +; CHECK-GI-NEXT: ldr w9, [sp, #272] +; CHECK-GI-NEXT: ldr w11, [sp, #48] +; CHECK-GI-NEXT: mov v5.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #40] +; CHECK-GI-NEXT: fmov s18, w16 +; CHECK-GI-NEXT: fmov s0, w9 +; CHECK-GI-NEXT: ldr w9, [sp, #168] +; CHECK-GI-NEXT: mov v2.b[3], w3 +; CHECK-GI-NEXT: mov v3.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #232] +; CHECK-GI-NEXT: ldr w16, [sp, #584] +; CHECK-GI-NEXT: mov v6.b[2], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #104] +; CHECK-GI-NEXT: mov v5.b[3], w9 ; CHECK-GI-NEXT: ldr w9, [sp, #56] -; CHECK-GI-NEXT: ldr w12, [sp, #1072] -; CHECK-GI-NEXT: mov v3.b[12], w8 +; CHECK-GI-NEXT: mov v4.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #280] +; CHECK-GI-NEXT: mov v2.b[4], w4 +; CHECK-GI-NEXT: mov v3.b[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #240] +; CHECK-GI-NEXT: mov v6.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #112] +; CHECK-GI-NEXT: mov v0.b[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #176] +; CHECK-GI-NEXT: mov v4.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #288] +; CHECK-GI-NEXT: mov v2.b[5], w5 +; CHECK-GI-NEXT: mov v5.b[4], w10 +; CHECK-GI-NEXT: mov v3.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #248] +; CHECK-GI-NEXT: mov v0.b[2], w8 +; CHECK-GI-NEXT: mov v6.b[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #120] ; CHECK-GI-NEXT: ldr w8, [sp, #184] -; CHECK-GI-NEXT: mov v2.b[12], w11 -; CHECK-GI-NEXT: mov v0.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #824] -; CHECK-GI-NEXT: ldr w11, [sp, #312] -; CHECK-GI-NEXT: mov v4.b[12], w10 -; CHECK-GI-NEXT: mov v1.b[13], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #952] -; CHECK-GI-NEXT: mov v5.b[12], w12 -; CHECK-GI-NEXT: ldr w8, [sp, #64] -; CHECK-GI-NEXT: ldr w12, [sp, #1080] -; CHECK-GI-NEXT: mov v3.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #192] -; CHECK-GI-NEXT: mov v2.b[13], w11 -; CHECK-GI-NEXT: mov v0.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #832] -; CHECK-GI-NEXT: ldr w11, [sp, #320] -; CHECK-GI-NEXT: mov v4.b[13], w10 -; CHECK-GI-NEXT: mov v1.b[14], w9 -; CHECK-GI-NEXT: ldr w10, [sp, #960] -; CHECK-GI-NEXT: mov v5.b[13], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #72] -; CHECK-GI-NEXT: ldr w12, [sp, #1088] -; CHECK-GI-NEXT: mov v3.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #200] -; CHECK-GI-NEXT: mov v2.b[14], w11 -; CHECK-GI-NEXT: mov v0.b[15], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #72] +; CHECK-GI-NEXT: mov v4.b[5], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #136] +; CHECK-GI-NEXT: mov v2.b[6], w6 +; CHECK-GI-NEXT: mov v5.b[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #192] +; CHECK-GI-NEXT: mov v3.b[6], w13 +; CHECK-GI-NEXT: mov v0.b[3], w12 +; CHECK-GI-NEXT: mov v6.b[5], w9 +; CHECK-GI-NEXT: ldr w13, [sp, #304] +; CHECK-GI-NEXT: ldr w9, [sp, #256] +; CHECK-GI-NEXT: ldr w12, [sp, #200] +; CHECK-GI-NEXT: mov v4.b[6], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #336] +; CHECK-GI-NEXT: mov v2.b[7], w7 +; CHECK-GI-NEXT: mov v5.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #344] +; CHECK-GI-NEXT: mov v3.b[7], w10 +; CHECK-GI-NEXT: mov v0.b[4], w13 +; CHECK-GI-NEXT: fmov s7, w14 +; CHECK-GI-NEXT: mov v6.b[6], w9 +; CHECK-GI-NEXT: ldr w14, [sp, #312] +; CHECK-GI-NEXT: ldr w13, [sp, #352] +; CHECK-GI-NEXT: ldr w10, [sp, #360] +; CHECK-GI-NEXT: mov v4.b[7], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #472] +; CHECK-GI-NEXT: ldr w9, [sp, #320] +; CHECK-GI-NEXT: mov v5.b[7], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #400] +; CHECK-GI-NEXT: mov v7.b[1], w8 +; CHECK-GI-NEXT: mov v0.b[5], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #464] +; CHECK-GI-NEXT: mov v6.b[7], w15 +; CHECK-GI-NEXT: fmov s16, w12 +; CHECK-GI-NEXT: ldr w15, [sp, #408] +; CHECK-GI-NEXT: ldr w8, [sp, #328] +; CHECK-GI-NEXT: fmov s17, w14 +; CHECK-GI-NEXT: ldr w14, [sp, #592] +; CHECK-GI-NEXT: ldr w12, [sp, #368] +; CHECK-GI-NEXT: mov v7.b[2], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #536] +; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 +; CHECK-GI-NEXT: mov v16.b[1], w15 +; CHECK-GI-NEXT: fmov s19, w14 +; CHECK-GI-NEXT: ldr w15, [sp, #600] +; CHECK-GI-NEXT: mov v18.b[1], w13 +; CHECK-GI-NEXT: ldr w14, [sp, #416] +; CHECK-GI-NEXT: mov v17.b[1], w11 +; CHECK-GI-NEXT: mov v0.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #480] +; CHECK-GI-NEXT: ldr w11, [sp, #376] +; CHECK-GI-NEXT: mov v19.b[1], w15 +; CHECK-GI-NEXT: mov v7.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #544] +; CHECK-GI-NEXT: mov v16.b[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #608] +; CHECK-GI-NEXT: ldr w13, [sp, #384] +; CHECK-GI-NEXT: mov v18.b[2], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #424] +; CHECK-GI-NEXT: mov v17.b[2], w9 +; CHECK-GI-NEXT: mov v0.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #488] +; CHECK-GI-NEXT: ldr w9, [sp, #392] +; CHECK-GI-NEXT: mov v19.b[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #552] +; CHECK-GI-NEXT: mov v7.b[4], w12 +; CHECK-GI-NEXT: mov v16.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #656] +; CHECK-GI-NEXT: ldr w12, [sp, #616] +; CHECK-GI-NEXT: mov v17.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #432] +; CHECK-GI-NEXT: mov v18.b[3], w14 +; CHECK-GI-NEXT: fmov s1, w10 +; CHECK-GI-NEXT: ldr w14, [sp, #664] +; CHECK-GI-NEXT: ldr w10, [sp, #440] +; CHECK-GI-NEXT: mov v19.b[3], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #496] +; CHECK-GI-NEXT: mov v7.b[5], w11 +; CHECK-GI-NEXT: mov v16.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #624] +; CHECK-GI-NEXT: ldr w11, [sp, #504] +; CHECK-GI-NEXT: mov v1.b[1], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #560] +; CHECK-GI-NEXT: mov v17.b[4], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #448] +; CHECK-GI-NEXT: ldr w15, [sp, #456] +; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 +; CHECK-GI-NEXT: mov v18.b[4], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #672] +; CHECK-GI-NEXT: mov v19.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #568] +; CHECK-GI-NEXT: mov v16.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #632] +; CHECK-GI-NEXT: mov v1.b[2], w14 +; CHECK-GI-NEXT: mov v17.b[5], w11 +; CHECK-GI-NEXT: mov v7.b[6], w13 +; CHECK-GI-NEXT: ldr w11, [sp, #512] +; CHECK-GI-NEXT: ldr w13, [sp, #520] +; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 +; CHECK-GI-NEXT: mov v18.b[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #680] +; CHECK-GI-NEXT: mov v19.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #576] +; CHECK-GI-NEXT: mov v16.b[6], w12 +; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0 +; CHECK-GI-NEXT: mov v1.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #640] +; CHECK-GI-NEXT: mov v17.b[6], w11 +; CHECK-GI-NEXT: mov v7.b[7], w9 +; CHECK-GI-NEXT: ushll v20.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: mov v18.b[6], w10 +; CHECK-GI-NEXT: mov v19.b[6], w8 +; CHECK-GI-NEXT: ldr w10, [sp, #688] +; CHECK-GI-NEXT: mov v16.b[7], w15 +; CHECK-GI-NEXT: ushll v21.4s, v3.4h, #0 +; CHECK-GI-NEXT: ushll2 v22.4s, v3.8h, #0 +; CHECK-GI-NEXT: mov v17.b[7], w13 +; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0 +; CHECK-GI-NEXT: ushll2 v24.4s, v4.8h, #0 +; CHECK-GI-NEXT: sshll v7.8h, v7.8b, #0 +; CHECK-GI-NEXT: ushll v3.4s, v6.4h, #0 +; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0 +; CHECK-GI-NEXT: mov v18.b[7], w16 +; CHECK-GI-NEXT: mov v19.b[7], w17 +; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 +; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0 +; CHECK-GI-NEXT: mov v1.b[4], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #696] +; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0 +; CHECK-GI-NEXT: ldr w14, [sp, #720] +; CHECK-GI-NEXT: ldr w13, [sp, #728] +; CHECK-GI-NEXT: ushll v25.4s, v5.4h, #0 +; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 +; CHECK-GI-NEXT: ldr w12, [sp, #704] +; CHECK-GI-NEXT: sshll v6.8h, v18.8b, #0 +; CHECK-GI-NEXT: sshll v18.8h, v19.8b, #0 +; CHECK-GI-NEXT: sshll v19.4s, v7.4h, #0 +; CHECK-GI-NEXT: sshll2 v7.4s, v7.8h, #0 +; CHECK-GI-NEXT: sshll v26.4s, v16.4h, #0 +; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0 +; CHECK-GI-NEXT: sshll v27.4s, v17.4h, #0 +; CHECK-GI-NEXT: mov v1.b[5], w9 +; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0 +; CHECK-GI-NEXT: mul v19.4s, v20.4s, v19.4s +; CHECK-GI-NEXT: sshll v28.4s, v6.4h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v6.8h, #0 +; CHECK-GI-NEXT: mul v7.4s, v2.4s, v7.4s +; CHECK-GI-NEXT: mul v20.4s, v21.4s, v26.4s +; CHECK-GI-NEXT: mul v16.4s, v22.4s, v16.4s +; CHECK-GI-NEXT: mul v21.4s, v23.4s, v27.4s +; CHECK-GI-NEXT: fmov s2, w14 +; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0 +; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0 +; CHECK-GI-NEXT: mul v5.4s, v5.4s, v6.4s +; CHECK-GI-NEXT: mul v17.4s, v24.4s, v17.4s +; CHECK-GI-NEXT: addv s6, v19.4s +; CHECK-GI-NEXT: mov v1.b[6], w12 +; CHECK-GI-NEXT: ldr w11, [sp, #736] +; CHECK-GI-NEXT: addv s7, v7.4s +; CHECK-GI-NEXT: mov v2.b[1], w13 +; CHECK-GI-NEXT: addv s16, v16.4s +; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s +; CHECK-GI-NEXT: addv s18, v20.4s +; CHECK-GI-NEXT: addv s19, v21.4s +; CHECK-GI-NEXT: fmov w12, s6 +; CHECK-GI-NEXT: mul v22.4s, v25.4s, v28.4s +; CHECK-GI-NEXT: addv s17, v17.4s +; CHECK-GI-NEXT: fmov w13, s7 +; CHECK-GI-NEXT: mul v3.4s, v3.4s, v29.4s +; CHECK-GI-NEXT: addv s5, v5.4s +; CHECK-GI-NEXT: mov v2.b[2], w11 +; CHECK-GI-NEXT: fmov w14, s19 +; CHECK-GI-NEXT: ldr w10, [sp, #744] +; CHECK-GI-NEXT: ldr w9, [sp, #752] +; CHECK-GI-NEXT: ldr w8, [sp, #712] +; CHECK-GI-NEXT: ldr w15, [sp, #864] +; CHECK-GI-NEXT: add w11, w12, w13 +; CHECK-GI-NEXT: fmov w12, s18 +; CHECK-GI-NEXT: fmov w13, s16 +; CHECK-GI-NEXT: addv s20, v22.4s +; CHECK-GI-NEXT: addv s3, v3.4s +; CHECK-GI-NEXT: mov v1.b[7], w8 +; CHECK-GI-NEXT: mov v2.b[3], w10 +; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 +; CHECK-GI-NEXT: add w11, w11, w12 +; CHECK-GI-NEXT: add w12, w13, w14 +; CHECK-GI-NEXT: fmov w13, s17 +; CHECK-GI-NEXT: fmov w10, s20 +; CHECK-GI-NEXT: ldr w14, [sp, #784] +; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll v22.4s, v0.4h, #0 +; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 +; CHECK-GI-NEXT: add w12, w12, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #792] +; CHECK-GI-NEXT: mov v2.b[4], w9 +; CHECK-GI-NEXT: add w11, w11, w12 +; CHECK-GI-NEXT: fmov w12, s5 +; CHECK-GI-NEXT: addv s5, v4.4s +; CHECK-GI-NEXT: ldr w9, [sp, #856] +; CHECK-GI-NEXT: sshll v23.4s, v1.4h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 +; CHECK-GI-NEXT: add w10, w10, w12 +; CHECK-GI-NEXT: fmov w12, s3 +; CHECK-GI-NEXT: fmov s3, w14 +; CHECK-GI-NEXT: ldr w14, [sp, #760] +; CHECK-GI-NEXT: mul v22.4s, v22.4s, v23.4s +; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: add w12, w10, w12 +; CHECK-GI-NEXT: ldr w10, [sp, #848] +; CHECK-GI-NEXT: mov v3.b[1], w13 +; CHECK-GI-NEXT: fmov w13, s5 +; CHECK-GI-NEXT: mov v2.b[5], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #920] +; CHECK-GI-NEXT: fmov s4, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #768] +; CHECK-GI-NEXT: addv s1, v22.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: add w12, w12, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #800] +; CHECK-GI-NEXT: mov v4.b[1], w9 +; CHECK-GI-NEXT: add w8, w11, w12 +; CHECK-GI-NEXT: ldr w12, [sp, #912] +; CHECK-GI-NEXT: mov v3.b[2], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #976] +; CHECK-GI-NEXT: ldr w11, [sp, #984] +; CHECK-GI-NEXT: fmov s5, w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1048] +; CHECK-GI-NEXT: mov v2.b[6], w10 +; CHECK-GI-NEXT: fmov s6, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #808] +; CHECK-GI-NEXT: ldr w10, [sp, #832] +; CHECK-GI-NEXT: mov v4.b[2], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #1040] +; CHECK-GI-NEXT: ldr w9, [sp, #776] +; CHECK-GI-NEXT: mov v5.b[1], w14 +; CHECK-GI-NEXT: mov v3.b[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1056] +; CHECK-GI-NEXT: fmov s7, w15 +; CHECK-GI-NEXT: mov v6.b[1], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #872] +; CHECK-GI-NEXT: ldr w14, [sp, #816] +; CHECK-GI-NEXT: ldr w15, [sp, #992] +; CHECK-GI-NEXT: mov v2.b[7], w9 +; CHECK-GI-NEXT: mov v4.b[3], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #824] ; CHECK-GI-NEXT: ldr w9, [sp, #840] -; CHECK-GI-NEXT: ldr w11, [sp, #328] -; CHECK-GI-NEXT: mov v4.b[14], w10 -; CHECK-GI-NEXT: mov v1.b[15], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #968] -; CHECK-GI-NEXT: mov v5.b[14], w12 -; CHECK-GI-NEXT: ldr w10, [sp, #1096] -; CHECK-GI-NEXT: mov v3.b[15], w9 -; CHECK-GI-NEXT: mov v2.b[15], w11 -; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b -; CHECK-GI-NEXT: mov v4.b[15], w8 -; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b -; CHECK-GI-NEXT: mov v5.b[15], w10 -; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b -; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b -; CHECK-GI-NEXT: addv s0, v7.4s -; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b -; CHECK-GI-NEXT: addv s1, v16.4s -; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b -; CHECK-GI-NEXT: addv s2, v17.4s -; CHECK-GI-NEXT: addv s4, v20.4s -; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: mov v7.b[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #928] +; CHECK-GI-NEXT: mov v3.b[4], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1064] +; CHECK-GI-NEXT: mov v6.b[2], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #1000] +; CHECK-GI-NEXT: mov v5.b[2], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #880] +; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 +; CHECK-GI-NEXT: mov v7.b[2], w13 +; CHECK-GI-NEXT: mov v4.b[4], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1104] +; CHECK-GI-NEXT: ldr w13, [sp, #936] +; CHECK-GI-NEXT: mov v3.b[5], w11 +; CHECK-GI-NEXT: mov v6.b[3], w15 +; CHECK-GI-NEXT: fmov s16, w12 +; CHECK-GI-NEXT: ldr w12, [sp, #944] +; CHECK-GI-NEXT: ldr w11, [sp, #1008] +; CHECK-GI-NEXT: mov v5.b[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1112] +; CHECK-GI-NEXT: ldr w15, [sp, #904] +; CHECK-GI-NEXT: mov v7.b[3], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #888] +; CHECK-GI-NEXT: mov v16.b[1], w13 +; CHECK-GI-NEXT: mov v3.b[6], w10 +; CHECK-GI-NEXT: ldr w13, [sp, #896] +; CHECK-GI-NEXT: mov v4.b[5], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1072] +; CHECK-GI-NEXT: mov v6.b[4], w11 +; CHECK-GI-NEXT: mov v5.b[4], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1120] +; CHECK-GI-NEXT: ldr w11, [sp, #952] +; CHECK-GI-NEXT: mov v7.b[4], w14 +; CHECK-GI-NEXT: ldr w10, [sp, #1016] +; CHECK-GI-NEXT: ldr w14, [sp, #960] +; CHECK-GI-NEXT: mov v16.b[2], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1080] +; CHECK-GI-NEXT: mov v3.b[7], w9 +; CHECK-GI-NEXT: mov v4.b[6], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1128] +; CHECK-GI-NEXT: ldr w9, [sp, #1088] +; CHECK-GI-NEXT: mov v5.b[5], w11 +; CHECK-GI-NEXT: mov v6.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1024] +; CHECK-GI-NEXT: mov v7.b[5], w12 +; CHECK-GI-NEXT: ldr w11, [sp, #968] +; CHECK-GI-NEXT: ldr w12, [sp, #1032] +; CHECK-GI-NEXT: mov v16.b[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1136] +; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 +; CHECK-GI-NEXT: mov v4.b[7], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #1144] +; CHECK-GI-NEXT: mov v5.b[6], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1168] +; CHECK-GI-NEXT: mov v6.b[6], w10 +; CHECK-GI-NEXT: mov v7.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1232] +; CHECK-GI-NEXT: ldr w10, [sp, #1176] +; CHECK-GI-NEXT: mov v16.b[4], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1240] +; CHECK-GI-NEXT: fmov s17, w14 +; CHECK-GI-NEXT: fmov s18, w9 +; CHECK-GI-NEXT: ldr w14, [sp, #1096] +; CHECK-GI-NEXT: ldr w9, [sp, #1160] +; CHECK-GI-NEXT: mov v5.b[7], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1184] +; CHECK-GI-NEXT: mov v6.b[7], w12 +; CHECK-GI-NEXT: mov v17.b[1], w10 +; CHECK-GI-NEXT: ldr w12, [sp, #1248] +; CHECK-GI-NEXT: mov v7.b[7], w14 +; CHECK-GI-NEXT: mov v18.b[1], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1296] +; CHECK-GI-NEXT: mov v16.b[5], w15 +; CHECK-GI-NEXT: ldr w15, [sp, #1360] +; CHECK-GI-NEXT: ldr w14, [sp, #1304] +; CHECK-GI-NEXT: ldr w10, [sp, #1152] +; CHECK-GI-NEXT: fmov s19, w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1424] +; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 +; CHECK-GI-NEXT: mov v17.b[2], w11 +; CHECK-GI-NEXT: fmov s20, w15 +; CHECK-GI-NEXT: ldr w11, [sp, #1368] +; CHECK-GI-NEXT: fmov s21, w13 +; CHECK-GI-NEXT: mov v18.b[2], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1432] +; CHECK-GI-NEXT: mov v19.b[1], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1192] +; CHECK-GI-NEXT: ldr w13, [sp, #1200] +; CHECK-GI-NEXT: mov v20.b[1], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1312] +; CHECK-GI-NEXT: mov v16.b[6], w10 +; CHECK-GI-NEXT: mov v21.b[1], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1256] +; CHECK-GI-NEXT: mov v17.b[3], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1440] +; CHECK-GI-NEXT: ldr w10, [sp, #1376] +; CHECK-GI-NEXT: ushll v22.4s, v3.4h, #0 +; CHECK-GI-NEXT: mov v18.b[3], w12 +; CHECK-GI-NEXT: mov v19.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1208] +; CHECK-GI-NEXT: mov v20.b[2], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1320] +; CHECK-GI-NEXT: mov v16.b[7], w9 +; CHECK-GI-NEXT: mov v21.b[2], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1264] +; CHECK-GI-NEXT: mov v17.b[4], w13 +; CHECK-GI-NEXT: ldr w9, [sp, #1384] +; CHECK-GI-NEXT: ldr w13, [sp, #1448] +; CHECK-GI-NEXT: ldr w12, [sp, #1216] +; CHECK-GI-NEXT: mov v18.b[4], w14 +; CHECK-GI-NEXT: mov v19.b[3], w10 +; CHECK-GI-NEXT: ldr w14, [sp, #1272] +; CHECK-GI-NEXT: mov v20.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1328] +; CHECK-GI-NEXT: ldr w10, [sp, #1224] +; CHECK-GI-NEXT: mov v17.b[5], w11 +; CHECK-GI-NEXT: mov v21.b[3], w13 +; CHECK-GI-NEXT: ldr w13, [sp, #1280] +; CHECK-GI-NEXT: ldr w11, [sp, #1392] +; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0 +; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0 +; CHECK-GI-NEXT: mov v18.b[5], w14 +; CHECK-GI-NEXT: mov v19.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1336] +; CHECK-GI-NEXT: ldr w14, [sp, #1456] +; CHECK-GI-NEXT: mov v20.b[4], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1288] +; CHECK-GI-NEXT: mov v17.b[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1344] +; CHECK-GI-NEXT: sshll v27.4s, v16.4h, #0 +; CHECK-GI-NEXT: mov v21.b[4], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1400] +; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0 +; CHECK-GI-NEXT: mov v18.b[6], w13 +; CHECK-GI-NEXT: mov v19.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1464] +; CHECK-GI-NEXT: mov v20.b[5], w14 +; CHECK-GI-NEXT: ldr w14, [sp, #1352] +; CHECK-GI-NEXT: ldr w13, [sp, #1416] +; CHECK-GI-NEXT: mov v17.b[7], w10 +; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0 +; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 +; CHECK-GI-NEXT: mov v21.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1408] +; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0 +; CHECK-GI-NEXT: mov v18.b[7], w11 +; CHECK-GI-NEXT: mov v19.b[6], w12 +; CHECK-GI-NEXT: ldr w12, [sp, #1472] +; CHECK-GI-NEXT: mov v20.b[6], w9 ; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: addv s3, v18.4s -; CHECK-GI-NEXT: addv s5, v19.4s -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s4 -; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: ushll v1.4s, v2.4h, #0 +; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0 +; CHECK-GI-NEXT: ldr w15, [sp, #1480] +; CHECK-GI-NEXT: mov v21.b[6], w12 +; CHECK-GI-NEXT: ushll v24.4s, v5.4h, #0 +; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0 +; CHECK-GI-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-GI-NEXT: mov v19.b[7], w14 +; CHECK-GI-NEXT: mul v1.4s, v1.4s, v27.4s +; CHECK-GI-NEXT: sshll v28.4s, v17.4h, #0 +; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0 +; CHECK-GI-NEXT: mul v2.4s, v2.4s, v16.4s +; CHECK-GI-NEXT: mov v20.b[7], w13 +; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 +; CHECK-GI-NEXT: ushll v25.4s, v6.4h, #0 +; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0 +; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0 +; CHECK-GI-NEXT: mov v21.b[7], w15 +; CHECK-GI-NEXT: sshll v19.8h, v19.8b, #0 +; CHECK-GI-NEXT: mul v16.4s, v22.4s, v28.4s +; CHECK-GI-NEXT: mul v3.4s, v3.4s, v17.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: addv s2, v2.4s +; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 +; CHECK-GI-NEXT: mul v17.4s, v23.4s, v29.4s +; CHECK-GI-NEXT: sshll v20.8h, v20.8b, #0 +; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s +; CHECK-GI-NEXT: sshll v30.4s, v19.4h, #0 +; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0 +; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0 +; CHECK-GI-NEXT: addv s16, v16.4s +; CHECK-GI-NEXT: addv s3, v3.4s +; CHECK-GI-NEXT: fmov w10, s1 +; CHECK-GI-NEXT: fmov w11, s2 +; CHECK-GI-NEXT: sshll v31.4s, v20.4h, #0 +; CHECK-GI-NEXT: sshll v21.8h, v21.8b, #0 +; CHECK-GI-NEXT: addv s17, v17.4s +; CHECK-GI-NEXT: mul v18.4s, v24.4s, v30.4s +; CHECK-GI-NEXT: mul v5.4s, v5.4s, v19.4s +; CHECK-GI-NEXT: addv s4, v4.4s +; CHECK-GI-NEXT: fmov w12, s3 +; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0 +; CHECK-GI-NEXT: add w10, w10, w11 +; CHECK-GI-NEXT: fmov w11, s16 +; CHECK-GI-NEXT: mul v19.4s, v25.4s, v31.4s +; CHECK-GI-NEXT: fmov w13, s17 +; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0 +; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0 +; CHECK-GI-NEXT: addv s18, v18.4s +; CHECK-GI-NEXT: addv s5, v5.4s +; CHECK-GI-NEXT: sshll v27.4s, v21.4h, #0 +; CHECK-GI-NEXT: add w10, w10, w11 +; CHECK-GI-NEXT: sshll2 v21.4s, v21.8h, #0 +; CHECK-GI-NEXT: mul v6.4s, v6.4s, v20.4s +; CHECK-GI-NEXT: add w11, w12, w13 +; CHECK-GI-NEXT: fmov w12, s4 +; CHECK-GI-NEXT: addv s19, v19.4s +; CHECK-GI-NEXT: fmov w13, s5 +; CHECK-GI-NEXT: mul v20.4s, v26.4s, v27.4s +; CHECK-GI-NEXT: fmov w14, s0 +; CHECK-GI-NEXT: mul v7.4s, v7.4s, v21.4s +; CHECK-GI-NEXT: add w11, w11, w12 +; CHECK-GI-NEXT: fmov w12, s18 +; CHECK-GI-NEXT: addv s6, v6.4s +; CHECK-GI-NEXT: add w10, w10, w11 +; CHECK-GI-NEXT: add w9, w9, w14 +; CHECK-GI-NEXT: addv s1, v20.4s ; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w12, w12, w13 +; CHECK-GI-NEXT: fmov w13, s19 +; CHECK-GI-NEXT: addv s2, v7.4s +; CHECK-GI-NEXT: add w11, w12, w13 +; CHECK-GI-NEXT: fmov w12, s6 +; CHECK-GI-NEXT: fmov w13, s2 +; CHECK-GI-NEXT: add w11, w11, w12 +; CHECK-GI-NEXT: fmov w12, s1 ; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s5 +; CHECK-GI-NEXT: add w11, w12, w13 ; CHECK-GI-NEXT: add w9, w10, w11 ; CHECK-GI-NEXT: add w0, w8, w9 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-GI-NEXT: ret entry: - %az = sext <48 x i8> %a to <48 x i32> - %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az) - %cz = sext <48 x i8> %c to <48 x i32> - %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz) + %az = zext <48 x i8> %a to <48 x i32> + %bz = sext <48 x i8> %b to <48 x i32> + %m1 = mul nuw nsw <48 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1) + %cz = zext <48 x i8> %c to <48 x i32> + %dz = sext <48 x i8> %d to <48 x i32> + %m2 = mul nuw nsw <48 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2) %x = add i32 %r1, %r2 ret i32 %x } @@ -4527,3 +7593,385 @@ entry: %x = add i32 %r1, %r2 ret i32 %x } + +define i32 @test_usdot_v64i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_usdot_v64i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: ldp q1, q2, [x0, #32] +; CHECK-SD-NEXT: ldp q6, q7, [x1, #32] +; CHECK-SD-NEXT: ldp q16, q17, [x0] +; CHECK-SD-NEXT: ldp q18, q19, [x1] +; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v7.16b +; CHECK-SD-NEXT: usdot v5.4s, v1.16b, v6.16b +; CHECK-SD-NEXT: usdot v4.4s, v17.16b, v19.16b +; CHECK-SD-NEXT: usdot v3.4s, v16.16b, v18.16b +; CHECK-SD-NEXT: add v0.4s, v4.4s, v0.4s +; CHECK-SD-NEXT: add v1.4s, v3.4s, v5.4s +; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v64i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: stp d15, d14, [sp, #-64]! // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d13, d12, [sp, #16] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d11, d10, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d9, d8, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 64 +; CHECK-GI-NEXT: .cfi_offset b8, -8 +; CHECK-GI-NEXT: .cfi_offset b9, -16 +; CHECK-GI-NEXT: .cfi_offset b10, -24 +; CHECK-GI-NEXT: .cfi_offset b11, -32 +; CHECK-GI-NEXT: .cfi_offset b12, -40 +; CHECK-GI-NEXT: .cfi_offset b13, -48 +; CHECK-GI-NEXT: .cfi_offset b14, -56 +; CHECK-GI-NEXT: .cfi_offset b15, -64 +; CHECK-GI-NEXT: ldp q0, q1, [x1] +; CHECK-GI-NEXT: ldp q21, q17, [x0] +; CHECK-GI-NEXT: ldp q3, q19, [x1, #32] +; CHECK-GI-NEXT: ldp q18, q4, [x0, #32] +; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v5.8h, v0.16b, #0 +; CHECK-GI-NEXT: sshll v7.8h, v1.8b, #0 +; CHECK-GI-NEXT: sshll2 v22.8h, v1.16b, #0 +; CHECK-GI-NEXT: sshll v23.8h, v3.8b, #0 +; CHECK-GI-NEXT: sshll2 v24.8h, v3.16b, #0 +; CHECK-GI-NEXT: sshll v25.8h, v19.8b, #0 +; CHECK-GI-NEXT: sshll2 v26.8h, v19.16b, #0 +; CHECK-GI-NEXT: ushll v27.8h, v21.8b, #0 +; CHECK-GI-NEXT: ushll2 v28.8h, v21.16b, #0 +; CHECK-GI-NEXT: ushll v30.8h, v17.8b, #0 +; CHECK-GI-NEXT: ushll2 v17.8h, v17.16b, #0 +; CHECK-GI-NEXT: ushll v8.8h, v18.8b, #0 +; CHECK-GI-NEXT: ushll2 v18.8h, v18.16b, #0 +; CHECK-GI-NEXT: ushll v9.8h, v4.8b, #0 +; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0 +; CHECK-GI-NEXT: sshll v0.4s, v2.4h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v2.8h, #0 +; CHECK-GI-NEXT: sshll v1.4s, v5.4h, #0 +; CHECK-GI-NEXT: sshll2 v16.4s, v5.8h, #0 +; CHECK-GI-NEXT: sshll v2.4s, v7.4h, #0 +; CHECK-GI-NEXT: sshll2 v20.4s, v7.8h, #0 +; CHECK-GI-NEXT: sshll v3.4s, v22.4h, #0 +; CHECK-GI-NEXT: sshll2 v22.4s, v22.8h, #0 +; CHECK-GI-NEXT: sshll v5.4s, v23.4h, #0 +; CHECK-GI-NEXT: sshll2 v23.4s, v23.8h, #0 +; CHECK-GI-NEXT: sshll v7.4s, v24.4h, #0 +; CHECK-GI-NEXT: sshll2 v24.4s, v24.8h, #0 +; CHECK-GI-NEXT: sshll v19.4s, v25.4h, #0 +; CHECK-GI-NEXT: sshll2 v25.4s, v25.8h, #0 +; CHECK-GI-NEXT: sshll v21.4s, v26.4h, #0 +; CHECK-GI-NEXT: sshll2 v26.4s, v26.8h, #0 +; CHECK-GI-NEXT: ushll v29.4s, v27.4h, #0 +; CHECK-GI-NEXT: ushll2 v27.4s, v27.8h, #0 +; CHECK-GI-NEXT: ushll v31.4s, v28.4h, #0 +; CHECK-GI-NEXT: ushll2 v28.4s, v28.8h, #0 +; CHECK-GI-NEXT: ushll v10.4s, v30.4h, #0 +; CHECK-GI-NEXT: ushll2 v30.4s, v30.8h, #0 +; CHECK-GI-NEXT: ushll v11.4s, v17.4h, #0 +; CHECK-GI-NEXT: ushll2 v17.4s, v17.8h, #0 +; CHECK-GI-NEXT: ushll2 v12.4s, v8.8h, #0 +; CHECK-GI-NEXT: ushll2 v13.4s, v18.8h, #0 +; CHECK-GI-NEXT: ushll2 v14.4s, v9.8h, #0 +; CHECK-GI-NEXT: ushll2 v15.4s, v4.8h, #0 +; CHECK-GI-NEXT: mul v6.4s, v6.4s, v27.4s +; CHECK-GI-NEXT: mul v16.4s, v16.4s, v28.4s +; CHECK-GI-NEXT: mul v20.4s, v20.4s, v30.4s +; CHECK-GI-NEXT: mul v17.4s, v22.4s, v17.4s +; CHECK-GI-NEXT: ushll v8.4s, v8.4h, #0 +; CHECK-GI-NEXT: mul v22.4s, v23.4s, v12.4s +; CHECK-GI-NEXT: mul v23.4s, v24.4s, v13.4s +; CHECK-GI-NEXT: mul v24.4s, v25.4s, v14.4s +; CHECK-GI-NEXT: ldp d13, d12, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: mul v25.4s, v26.4s, v15.4s +; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0 +; CHECK-GI-NEXT: ushll v26.4s, v9.4h, #0 +; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0 +; CHECK-GI-NEXT: mla v6.4s, v0.4s, v29.4s +; CHECK-GI-NEXT: mla v16.4s, v1.4s, v31.4s +; CHECK-GI-NEXT: mla v20.4s, v2.4s, v10.4s +; CHECK-GI-NEXT: mla v17.4s, v3.4s, v11.4s +; CHECK-GI-NEXT: ldp d11, d10, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: mla v22.4s, v5.4s, v8.4s +; CHECK-GI-NEXT: ldp d9, d8, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: mla v23.4s, v7.4s, v18.4s +; CHECK-GI-NEXT: mla v24.4s, v19.4s, v26.4s +; CHECK-GI-NEXT: mla v25.4s, v21.4s, v4.4s +; CHECK-GI-NEXT: add v0.4s, v6.4s, v16.4s +; CHECK-GI-NEXT: add v1.4s, v20.4s, v17.4s +; CHECK-GI-NEXT: add v2.4s, v22.4s, v23.4s +; CHECK-GI-NEXT: add v3.4s, v24.4s, v25.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: add v1.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ldp d15, d14, [sp], #64 // 16-byte Folded Reload +; CHECK-GI-NEXT: ret +entry: + %0 = load <64 x i8>, ptr %a + %1 = zext <64 x i8> %0 to <64 x i32> + %2 = load <64 x i8>, ptr %b + %3 = sext <64 x i8> %2 to <64 x i32> + %4 = mul nsw <64 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_usdot_v64i8_double(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) { +; CHECK-SD-LABEL: test_usdot_v64i8_double: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v18.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v21.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v22.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v23.2d, #0000000000000000 +; CHECK-SD-NEXT: ldp q16, q17, [sp, #64] +; CHECK-SD-NEXT: movi v24.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v25.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v26.2d, #0000000000000000 +; CHECK-SD-NEXT: movi v27.2d, #0000000000000000 +; CHECK-SD-NEXT: ldp q19, q20, [sp, #96] +; CHECK-SD-NEXT: usdot v18.4s, v3.16b, v7.16b +; CHECK-SD-NEXT: ldp q3, q7, [sp, #32] +; CHECK-SD-NEXT: usdot v21.4s, v1.16b, v5.16b +; CHECK-SD-NEXT: ldp q1, q5, [sp] +; CHECK-SD-NEXT: usdot v22.4s, v2.16b, v6.16b +; CHECK-SD-NEXT: usdot v23.4s, v0.16b, v4.16b +; CHECK-SD-NEXT: usdot v24.4s, v7.16b, v20.16b +; CHECK-SD-NEXT: usdot v27.4s, v3.16b, v19.16b +; CHECK-SD-NEXT: usdot v26.4s, v5.16b, v17.16b +; CHECK-SD-NEXT: usdot v25.4s, v1.16b, v16.16b +; CHECK-SD-NEXT: add v0.4s, v21.4s, v18.4s +; CHECK-SD-NEXT: add v1.4s, v23.4s, v22.4s +; CHECK-SD-NEXT: add v2.4s, v26.4s, v24.4s +; CHECK-SD-NEXT: add v3.4s, v25.4s, v27.4s +; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-SD-NEXT: add v1.4s, v3.4s, v2.4s +; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_usdot_v64i8_double: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: sub sp, sp, #304 +; CHECK-GI-NEXT: stp d15, d14, [sp, #224] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d13, d12, [sp, #240] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d11, d10, [sp, #256] // 16-byte Folded Spill +; CHECK-GI-NEXT: stp d9, d8, [sp, #272] // 16-byte Folded Spill +; CHECK-GI-NEXT: str x29, [sp, #288] // 8-byte Folded Spill +; CHECK-GI-NEXT: .cfi_def_cfa_offset 304 +; CHECK-GI-NEXT: .cfi_offset w29, -16 +; CHECK-GI-NEXT: .cfi_offset b8, -24 +; CHECK-GI-NEXT: .cfi_offset b9, -32 +; CHECK-GI-NEXT: .cfi_offset b10, -40 +; CHECK-GI-NEXT: .cfi_offset b11, -48 +; CHECK-GI-NEXT: .cfi_offset b12, -56 +; CHECK-GI-NEXT: .cfi_offset b13, -64 +; CHECK-GI-NEXT: .cfi_offset b14, -72 +; CHECK-GI-NEXT: .cfi_offset b15, -80 +; CHECK-GI-NEXT: ushll v17.8h, v0.8b, #0 +; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 +; CHECK-GI-NEXT: ldr x29, [sp, #288] // 8-byte Folded Reload +; CHECK-GI-NEXT: mov v20.16b, v3.16b +; CHECK-GI-NEXT: ushll v16.8h, v1.8b, #0 +; CHECK-GI-NEXT: ushll2 v18.8h, v1.16b, #0 +; CHECK-GI-NEXT: ushll v26.8h, v2.8b, #0 +; CHECK-GI-NEXT: ldp q27, q28, [sp, #304] +; CHECK-GI-NEXT: ushll2 v29.8h, v2.16b, #0 +; CHECK-GI-NEXT: ushll v2.4s, v17.4h, #0 +; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0 +; CHECK-GI-NEXT: sshll v8.8h, v4.8b, #0 +; CHECK-GI-NEXT: ldp q23, q21, [sp, #368] +; CHECK-GI-NEXT: sshll2 v9.8h, v4.16b, #0 +; CHECK-GI-NEXT: sshll2 v11.8h, v5.16b, #0 +; CHECK-GI-NEXT: mov v25.16b, v7.16b +; CHECK-GI-NEXT: ushll2 v19.4s, v17.8h, #0 +; CHECK-GI-NEXT: stp q1, q2, [sp, #192] // 32-byte Folded Spill +; CHECK-GI-NEXT: ushll2 v3.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll2 v17.4s, v18.8h, #0 +; CHECK-GI-NEXT: ldp q24, q22, [sp, #336] +; CHECK-GI-NEXT: sshll v10.8h, v5.8b, #0 +; CHECK-GI-NEXT: sshll v12.8h, v6.8b, #0 +; CHECK-GI-NEXT: sshll2 v13.8h, v6.16b, #0 +; CHECK-GI-NEXT: mov v2.16b, v20.16b +; CHECK-GI-NEXT: sshll2 v0.4s, v8.8h, #0 +; CHECK-GI-NEXT: sshll2 v4.4s, v9.8h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v11.8h, #0 +; CHECK-GI-NEXT: ushll2 v7.4s, v16.8h, #0 +; CHECK-GI-NEXT: ushll2 v31.4s, v29.8h, #0 +; CHECK-GI-NEXT: sshll2 v5.4s, v10.8h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v13.8h, #0 +; CHECK-GI-NEXT: ushll2 v30.4s, v26.8h, #0 +; CHECK-GI-NEXT: ushll v14.8h, v2.8b, #0 +; CHECK-GI-NEXT: mul v20.4s, v19.4s, v0.4s +; CHECK-GI-NEXT: mul v19.4s, v3.4s, v4.4s +; CHECK-GI-NEXT: sshll v0.8h, v25.8b, #0 +; CHECK-GI-NEXT: mul v4.4s, v17.4s, v6.4s +; CHECK-GI-NEXT: sshll2 v15.4s, v12.8h, #0 +; CHECK-GI-NEXT: ldp q17, q3, [sp, #400] +; CHECK-GI-NEXT: mul v5.4s, v7.4s, v5.4s +; CHECK-GI-NEXT: mul v7.4s, v31.4s, v1.4s +; CHECK-GI-NEXT: ushll2 v31.8h, v2.16b, #0 +; CHECK-GI-NEXT: sshll2 v25.8h, v25.16b, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v0.8h, #0 +; CHECK-GI-NEXT: ushll v2.4s, v14.4h, #0 +; CHECK-GI-NEXT: sshll v0.4s, v0.4h, #0 +; CHECK-GI-NEXT: str q3, [sp, #96] // 16-byte Folded Spill +; CHECK-GI-NEXT: ushll2 v3.4s, v14.8h, #0 +; CHECK-GI-NEXT: mul v6.4s, v30.4s, v15.4s +; CHECK-GI-NEXT: str q31, [sp, #160] // 16-byte Folded Spill +; CHECK-GI-NEXT: ushll v30.4s, v26.4h, #0 +; CHECK-GI-NEXT: sshll v26.4s, v8.4h, #0 +; CHECK-GI-NEXT: ushll v14.8h, v27.8b, #0 +; CHECK-GI-NEXT: ushll v15.4s, v29.4h, #0 +; CHECK-GI-NEXT: sshll v29.4s, v9.4h, #0 +; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s +; CHECK-GI-NEXT: ushll2 v3.4s, v31.8h, #0 +; CHECK-GI-NEXT: ushll v31.8h, v28.8b, #0 +; CHECK-GI-NEXT: ushll v16.4s, v16.4h, #0 +; CHECK-GI-NEXT: sshll v8.4s, v10.4h, #0 +; CHECK-GI-NEXT: sshll v9.4s, v11.4h, #0 +; CHECK-GI-NEXT: sshll v10.4s, v12.4h, #0 +; CHECK-GI-NEXT: sshll v11.4s, v13.4h, #0 +; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0 +; CHECK-GI-NEXT: stp q3, q25, [sp, #112] // 32-byte Folded Spill +; CHECK-GI-NEXT: ldr q3, [sp, #208] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll2 v28.8h, v28.16b, #0 +; CHECK-GI-NEXT: mla v1.4s, v2.4s, v0.4s +; CHECK-GI-NEXT: ushll2 v0.4s, v31.8h, #0 +; CHECK-GI-NEXT: mla v5.4s, v16.4s, v8.4s +; CHECK-GI-NEXT: mla v20.4s, v3.4s, v26.4s +; CHECK-GI-NEXT: sshll2 v3.4s, v25.8h, #0 +; CHECK-GI-NEXT: mla v6.4s, v30.4s, v10.4s +; CHECK-GI-NEXT: mla v7.4s, v15.4s, v11.4s +; CHECK-GI-NEXT: sshll v25.8h, v23.8b, #0 +; CHECK-GI-NEXT: mla v4.4s, v18.4s, v9.4s +; CHECK-GI-NEXT: ushll v30.8h, v22.8b, #0 +; CHECK-GI-NEXT: ushll2 v26.8h, v22.16b, #0 +; CHECK-GI-NEXT: sshll v22.8h, v21.8b, #0 +; CHECK-GI-NEXT: str q3, [sp, #32] // 16-byte Folded Spill +; CHECK-GI-NEXT: ldr q3, [sp, #192] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll2 v8.8h, v27.16b, #0 +; CHECK-GI-NEXT: str q1, [sp, #48] // 16-byte Folded Spill +; CHECK-GI-NEXT: ldr q9, [sp, #32] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll2 v1.4s, v14.8h, #0 +; CHECK-GI-NEXT: stp q7, q6, [sp, #64] // 32-byte Folded Spill +; CHECK-GI-NEXT: mla v19.4s, v3.4s, v29.4s +; CHECK-GI-NEXT: sshll2 v7.4s, v25.8h, #0 +; CHECK-GI-NEXT: str q5, [sp, #176] // 16-byte Folded Spill +; CHECK-GI-NEXT: ushll v29.8h, v24.8b, #0 +; CHECK-GI-NEXT: ushll2 v27.8h, v24.16b, #0 +; CHECK-GI-NEXT: stp q0, q1, [sp] // 32-byte Folded Spill +; CHECK-GI-NEXT: ldp q0, q16, [sp, #96] // 32-byte Folded Reload +; CHECK-GI-NEXT: str q4, [sp, #144] // 16-byte Folded Spill +; CHECK-GI-NEXT: sshll2 v24.8h, v23.16b, #0 +; CHECK-GI-NEXT: ushll2 v18.4s, v26.8h, #0 +; CHECK-GI-NEXT: stp q19, q20, [sp, #192] // 32-byte Folded Spill +; CHECK-GI-NEXT: sshll2 v20.8h, v21.16b, #0 +; CHECK-GI-NEXT: sshll v21.8h, v17.8b, #0 +; CHECK-GI-NEXT: sshll2 v19.8h, v17.16b, #0 +; CHECK-GI-NEXT: sshll2 v17.8h, v0.16b, #0 +; CHECK-GI-NEXT: mul v16.4s, v16.4s, v9.4s +; CHECK-GI-NEXT: ldr q9, [sp, #16] // 16-byte Folded Reload +; CHECK-GI-NEXT: sshll v23.8h, v0.8b, #0 +; CHECK-GI-NEXT: sshll2 v2.4s, v22.8h, #0 +; CHECK-GI-NEXT: ushll2 v12.4s, v27.8h, #0 +; CHECK-GI-NEXT: ushll v26.4s, v26.4h, #0 +; CHECK-GI-NEXT: ushll2 v10.4s, v28.8h, #0 +; CHECK-GI-NEXT: sshll2 v0.4s, v17.8h, #0 +; CHECK-GI-NEXT: mul v7.4s, v9.4s, v7.4s +; CHECK-GI-NEXT: ldr q9, [sp] // 16-byte Folded Reload +; CHECK-GI-NEXT: sshll2 v5.4s, v19.8h, #0 +; CHECK-GI-NEXT: sshll v17.4s, v17.4h, #0 +; CHECK-GI-NEXT: sshll2 v3.4s, v20.8h, #0 +; CHECK-GI-NEXT: mul v2.4s, v9.4s, v2.4s +; CHECK-GI-NEXT: ldr q9, [sp, #128] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll2 v15.4s, v8.8h, #0 +; CHECK-GI-NEXT: mul v0.4s, v18.4s, v0.4s +; CHECK-GI-NEXT: ldr q18, [sp, #160] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll2 v11.4s, v29.8h, #0 +; CHECK-GI-NEXT: sshll v9.4s, v9.4h, #0 +; CHECK-GI-NEXT: ushll2 v13.4s, v30.8h, #0 +; CHECK-GI-NEXT: sshll2 v1.4s, v24.8h, #0 +; CHECK-GI-NEXT: ushll v18.4s, v18.4h, #0 +; CHECK-GI-NEXT: sshll2 v4.4s, v21.8h, #0 +; CHECK-GI-NEXT: sshll2 v6.4s, v23.8h, #0 +; CHECK-GI-NEXT: mul v5.4s, v12.4s, v5.4s +; CHECK-GI-NEXT: ushll v27.4s, v27.4h, #0 +; CHECK-GI-NEXT: sshll v19.4s, v19.4h, #0 +; CHECK-GI-NEXT: mla v0.4s, v26.4s, v17.4s +; CHECK-GI-NEXT: mul v3.4s, v10.4s, v3.4s +; CHECK-GI-NEXT: mul v1.4s, v15.4s, v1.4s +; CHECK-GI-NEXT: mla v16.4s, v18.4s, v9.4s +; CHECK-GI-NEXT: ldp q18, q17, [sp, #192] // 32-byte Folded Reload +; CHECK-GI-NEXT: mul v4.4s, v11.4s, v4.4s +; CHECK-GI-NEXT: mul v6.4s, v13.4s, v6.4s +; CHECK-GI-NEXT: ushll v28.4s, v28.4h, #0 +; CHECK-GI-NEXT: ldp d13, d12, [sp, #240] // 16-byte Folded Reload +; CHECK-GI-NEXT: sshll v20.4s, v20.4h, #0 +; CHECK-GI-NEXT: ushll v10.4s, v14.4h, #0 +; CHECK-GI-NEXT: ldp d15, d14, [sp, #224] // 16-byte Folded Reload +; CHECK-GI-NEXT: ushll v8.4s, v8.4h, #0 +; CHECK-GI-NEXT: ushll v31.4s, v31.4h, #0 +; CHECK-GI-NEXT: ushll v29.4s, v29.4h, #0 +; CHECK-GI-NEXT: ushll v30.4s, v30.4h, #0 +; CHECK-GI-NEXT: sshll v25.4s, v25.4h, #0 +; CHECK-GI-NEXT: sshll v24.4s, v24.4h, #0 +; CHECK-GI-NEXT: sshll v22.4s, v22.4h, #0 +; CHECK-GI-NEXT: sshll v21.4s, v21.4h, #0 +; CHECK-GI-NEXT: sshll v23.4s, v23.4h, #0 +; CHECK-GI-NEXT: mla v5.4s, v27.4s, v19.4s +; CHECK-GI-NEXT: ldr q19, [sp, #144] // 16-byte Folded Reload +; CHECK-GI-NEXT: add v17.4s, v17.4s, v18.4s +; CHECK-GI-NEXT: ldr q18, [sp, #176] // 16-byte Folded Reload +; CHECK-GI-NEXT: mla v3.4s, v28.4s, v20.4s +; CHECK-GI-NEXT: mla v7.4s, v10.4s, v25.4s +; CHECK-GI-NEXT: ldp d11, d10, [sp, #256] // 16-byte Folded Reload +; CHECK-GI-NEXT: mla v1.4s, v8.4s, v24.4s +; CHECK-GI-NEXT: ldp d9, d8, [sp, #272] // 16-byte Folded Reload +; CHECK-GI-NEXT: add v18.4s, v18.4s, v19.4s +; CHECK-GI-NEXT: ldp q20, q19, [sp, #64] // 32-byte Folded Reload +; CHECK-GI-NEXT: mla v2.4s, v31.4s, v22.4s +; CHECK-GI-NEXT: mla v4.4s, v29.4s, v21.4s +; CHECK-GI-NEXT: mla v6.4s, v30.4s, v23.4s +; CHECK-GI-NEXT: add v1.4s, v7.4s, v1.4s +; CHECK-GI-NEXT: add v19.4s, v19.4s, v20.4s +; CHECK-GI-NEXT: ldr q20, [sp, #48] // 16-byte Folded Reload +; CHECK-GI-NEXT: add v2.4s, v2.4s, v3.4s +; CHECK-GI-NEXT: add v16.4s, v20.4s, v16.4s +; CHECK-GI-NEXT: add v3.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: add v0.4s, v6.4s, v0.4s +; CHECK-GI-NEXT: add v4.4s, v17.4s, v18.4s +; CHECK-GI-NEXT: add v1.4s, v1.4s, v2.4s +; CHECK-GI-NEXT: add v5.4s, v19.4s, v16.4s +; CHECK-GI-NEXT: add v0.4s, v3.4s, v0.4s +; CHECK-GI-NEXT: add v2.4s, v4.4s, v5.4s +; CHECK-GI-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: addv s1, v2.4s +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: fmov w8, s1 +; CHECK-GI-NEXT: fmov w9, s0 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: add sp, sp, #304 +; CHECK-GI-NEXT: ret +entry: + %az = zext <64 x i8> %a to <64 x i32> + %bz = sext <64 x i8> %b to <64 x i32> + %m1 = mul nuw nsw <64 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m1) + %cz = zext <64 x i8> %c to <64 x i32> + %dz = sext <64 x i8> %d to <64 x i32> + %m2 = mul nuw nsw <64 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %m2) + %x = add i32 %r1, %r2 + ret i32 %x +} From 31e138df235e5c42e67c4195e7ee8d55c0496a01 Mon Sep 17 00:00:00 2001 From: Igor Kirillov Date: Wed, 18 Dec 2024 13:40:45 +0000 Subject: [PATCH 2/2] Remove some tests --- llvm/test/CodeGen/AArch64/neon-dotreduce.ll | 5298 ++++++------------- 1 file changed, 1475 insertions(+), 3823 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll index 05ac2956da00c..748555d7bdfa1 100644 --- a/llvm/test/CodeGen/AArch64/neon-dotreduce.ll +++ b/llvm/test/CodeGen/AArch64/neon-dotreduce.ll @@ -7,22 +7,16 @@ ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v5i8_double_nomla -; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8 -; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v5i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v25i8_nomla ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v25i8_double_nomla -; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8 -; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v25i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_udot_v33i8_nomla ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8 ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double ; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_sdot_v33i8_double_nomla -; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8 -; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_usdot_v33i8_double declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>) declare i32 @llvm.vector.reduce.add.v5i32(<5 x i32>) @@ -542,65 +536,6 @@ entry: ret i32 %x } -define i32 @test_usdot_v5i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_usdot_v5i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldr d0, [x0] -; CHECK-NEXT: ldr d1, [x1] -; CHECK-NEXT: movi v3.2d, #0000000000000000 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h -; CHECK-NEXT: mov v3.s[0], v2.s[0] -; CHECK-NEXT: smlal v3.4s, v1.4h, v0.4h -; CHECK-NEXT: addv s0, v3.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret -entry: - %0 = load <5 x i8>, ptr %a - %1 = zext <5 x i8> %0 to <5 x i32> - %2 = load <5 x i8>, ptr %b - %3 = sext <5 x i8> %2 to <5 x i32> - %4 = mul nsw <5 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %4) - %op.extra = add nsw i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_usdot_v5i8_double(<5 x i8> %a, <5 x i8> %b, <5 x i8> %c, <5 x i8> %d) { -; CHECK-LABEL: test_usdot_v5i8_double: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v2.8h, v2.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: movi v6.2d, #0000000000000000 -; CHECK-NEXT: smull2 v4.4s, v0.8h, v1.8h -; CHECK-NEXT: smull2 v7.4s, v2.8h, v3.8h -; CHECK-NEXT: mov v6.s[0], v4.s[0] -; CHECK-NEXT: mov v5.s[0], v7.s[0] -; CHECK-NEXT: smlal v6.4s, v0.4h, v1.4h -; CHECK-NEXT: smlal v5.4s, v2.4h, v3.4h -; CHECK-NEXT: add v0.4s, v6.4s, v5.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ret -entry: - %az = zext <5 x i8> %a to <5 x i32> - %bz = sext <5 x i8> %b to <5 x i32> - %m1 = mul nuw nsw <5 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m1) - %cz = zext <5 x i8> %c to <5 x i32> - %dz = sext <5 x i8> %d to <5 x i32> - %m2 = mul nuw nsw <5 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v5i32(<5 x i32> %m2) - %x = add i32 %r1, %r2 - ret i32 %x -} - - define i32 @test_udot_v8i8(ptr nocapture readonly %a, ptr nocapture readonly %b) { ; CHECK-LABEL: test_udot_v8i8: ; CHECK: // %bb.0: // %entry @@ -2245,671 +2180,100 @@ entry: ret i32 %x } -define i32 @test_usdot_v24i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-SD-LABEL: test_usdot_v24i8: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 -; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 -; CHECK-SD-NEXT: ldr q2, [x0] -; CHECK-SD-NEXT: ldr q3, [x1] -; CHECK-SD-NEXT: ldr d4, [x1, #16] -; CHECK-SD-NEXT: ldr d5, [x0, #16] -; CHECK-SD-NEXT: usdot v1.2s, v5.8b, v4.8b -; CHECK-SD-NEXT: usdot v0.4s, v2.16b, v3.16b -; CHECK-SD-NEXT: addp v1.2s, v1.2s, v1.2s -; CHECK-SD-NEXT: addv s0, v0.4s -; CHECK-SD-NEXT: fmov w8, s1 -; CHECK-SD-NEXT: fmov w9, s0 -; CHECK-SD-NEXT: add w8, w9, w8 -; CHECK-SD-NEXT: add w0, w8, w2 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_usdot_v24i8: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ldr q0, [x0] -; CHECK-GI-NEXT: ldr d1, [x0, #16] -; CHECK-GI-NEXT: ldr q2, [x1] -; CHECK-GI-NEXT: ldr d3, [x1, #16] -; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0 -; CHECK-GI-NEXT: ushll2 v0.8h, v0.16b, #0 -; CHECK-GI-NEXT: sshll v5.8h, v2.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0 -; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-GI-NEXT: ushll v6.4s, v4.4h, #0 -; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0 -; CHECK-GI-NEXT: ushll v7.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: sshll v16.4s, v5.4h, #0 -; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0 -; CHECK-GI-NEXT: ushll v17.4s, v1.4h, #0 -; CHECK-GI-NEXT: sshll2 v18.4s, v2.8h, #0 -; CHECK-GI-NEXT: sshll v19.4s, v3.4h, #0 -; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: sshll v2.4s, v2.4h, #0 -; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 -; CHECK-GI-NEXT: mul v6.4s, v16.4s, v6.4s -; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s -; CHECK-GI-NEXT: mul v0.4s, v18.4s, v0.4s -; CHECK-GI-NEXT: mul v5.4s, v19.4s, v17.4s -; CHECK-GI-NEXT: mul v2.4s, v2.4s, v7.4s -; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s -; CHECK-GI-NEXT: addv s3, v6.4s -; CHECK-GI-NEXT: addv s4, v4.4s -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: addv s5, v5.4s -; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: fmov w8, s3 -; CHECK-GI-NEXT: fmov w9, s4 -; CHECK-GI-NEXT: fmov w10, s0 -; CHECK-GI-NEXT: fmov w11, s5 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s1 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w9, w10, w11 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ret +define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_udot_v25i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q3, q0, [x1] +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b +; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b +; CHECK-NEXT: ushll v3.4s, v4.4h, #0 +; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h +; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v5.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret entry: - %0 = load <24 x i8>, ptr %a - %1 = zext <24 x i8> %0 to <24 x i32> - %2 = load <24 x i8>, ptr %b - %3 = sext <24 x i8> %2 to <24 x i32> - %4 = mul nsw <24 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %4) + %0 = load <25 x i8>, ptr %a + %1 = zext <25 x i8> %0 to <25 x i32> + %2 = load <25 x i8>, ptr %b + %3 = zext <25 x i8> %2 to <25 x i32> + %4 = mul nuw nsw <25 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) + %op.extra = add i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) { +; CHECK-LABEL: test_udot_v25i8_nomla: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: movi v0.2d, #0000000000000000 +; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 +; CHECK-NEXT: ushll v1.8h, v1.8b, #0 +; CHECK-NEXT: ushll v4.8h, v2.8b, #0 +; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 +; CHECK-NEXT: ushll v3.4s, v3.4h, #0 +; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h +; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h +; CHECK-NEXT: mov v0.s[0], v3.s[0] +; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h +; CHECK-NEXT: add v1.4s, v1.4s, v3.4s +; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h +; CHECK-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret +entry: + %0 = load <25 x i8>, ptr %a1 + %1 = zext <25 x i8> %0 to <25 x i32> + %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1) + ret i32 %2 +} +define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-LABEL: test_sdot_v25i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: ldp q3, q0, [x1] +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: ldp q2, q1, [x0] +; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b +; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b +; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b +; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b +; CHECK-NEXT: sshll v3.4s, v4.4h, #0 +; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h +; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h +; CHECK-NEXT: mov v5.s[0], v3.s[0] +; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w8, s0 +; CHECK-NEXT: add w0, w8, w2 +; CHECK-NEXT: ret +entry: + %0 = load <25 x i8>, ptr %a + %1 = sext <25 x i8> %0 to <25 x i32> + %2 = load <25 x i8>, ptr %b + %3 = sext <25 x i8> %2 to <25 x i32> + %4 = mul nsw <25 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) %op.extra = add nsw i32 %5, %sum ret i32 %op.extra } -define i32 @test_usdot_v24i8_double(<24 x i8> %a, <24 x i8> %b, <24 x i8> %c, <24 x i8> %d) { -; CHECK-SD-LABEL: test_usdot_v24i8_double: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 -; CHECK-SD-NEXT: .cfi_offset w29, -16 -; CHECK-SD-NEXT: fmov s0, w0 -; CHECK-SD-NEXT: ldr b1, [sp, #144] -; CHECK-SD-NEXT: add x10, sp, #152 -; CHECK-SD-NEXT: add x9, sp, #160 -; CHECK-SD-NEXT: add x8, sp, #168 -; CHECK-SD-NEXT: ldr b2, [sp, #272] -; CHECK-SD-NEXT: ld1 { v1.b }[1], [x10] -; CHECK-SD-NEXT: add x11, sp, #280 -; CHECK-SD-NEXT: ldr b3, [sp, #80] -; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ldr b4, [sp, #528] -; CHECK-SD-NEXT: add x10, sp, #88 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x11] -; CHECK-SD-NEXT: add x11, sp, #536 -; CHECK-SD-NEXT: ldr b5, [sp, #336] -; CHECK-SD-NEXT: ld1 { v1.b }[2], [x9] -; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10] -; CHECK-SD-NEXT: add x10, sp, #344 -; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11] -; CHECK-SD-NEXT: add x11, sp, #176 -; CHECK-SD-NEXT: ldr b6, [sp, #656] -; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10] -; CHECK-SD-NEXT: ldr b7, [sp, #464] -; CHECK-SD-NEXT: ld1 { v1.b }[3], [x8] -; CHECK-SD-NEXT: add x12, sp, #664 -; CHECK-SD-NEXT: add x9, sp, #472 -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] -; CHECK-SD-NEXT: add x8, sp, #96 -; CHECK-SD-NEXT: add x10, sp, #184 -; CHECK-SD-NEXT: add x12, sp, #288 -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] -; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8] -; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v1.b }[4], [x11] -; CHECK-SD-NEXT: add x8, sp, #352 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12] -; CHECK-SD-NEXT: add x13, sp, #544 -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #672 -; CHECK-SD-NEXT: ld1 { v4.b }[2], [x13] -; CHECK-SD-NEXT: add x9, sp, #192 -; CHECK-SD-NEXT: ld1 { v1.b }[5], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #480 -; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #296 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #552 -; CHECK-SD-NEXT: add x12, sp, #200 -; CHECK-SD-NEXT: ld1 { v1.b }[6], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #360 -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #104 -; CHECK-SD-NEXT: add x9, sp, #560 -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #368 -; CHECK-SD-NEXT: ld1 { v1.b }[7], [x12] -; CHECK-SD-NEXT: ld1 { v4.b }[4], [x9] -; CHECK-SD-NEXT: add x13, sp, #208 -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-SD-NEXT: add x12, sp, #304 -; CHECK-SD-NEXT: add x8, sp, #568 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x12] -; CHECK-SD-NEXT: add x12, sp, #16 -; CHECK-SD-NEXT: add x17, sp, #376 -; CHECK-SD-NEXT: mov v0.b[6], w6 -; CHECK-SD-NEXT: ld1 { v1.b }[8], [x13] -; CHECK-SD-NEXT: ld1 { v4.b }[5], [x8] -; CHECK-SD-NEXT: add x14, sp, #216 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x17] -; CHECK-SD-NEXT: add x13, sp, #576 -; CHECK-SD-NEXT: add x11, sp, #224 -; CHECK-SD-NEXT: add x10, sp, #232 -; CHECK-SD-NEXT: add x15, sp, #240 -; CHECK-SD-NEXT: ld1 { v1.b }[9], [x14] -; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] -; CHECK-SD-NEXT: add x13, sp, #384 -; CHECK-SD-NEXT: mov v0.b[7], w7 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x13] -; CHECK-SD-NEXT: add x13, sp, #112 -; CHECK-SD-NEXT: ld1 { v3.b }[4], [x13] -; CHECK-SD-NEXT: add x13, sp, #32 -; CHECK-SD-NEXT: add x14, sp, #584 -; CHECK-SD-NEXT: ld1 { v1.b }[10], [x11] -; CHECK-SD-NEXT: ld1 { v4.b }[7], [x14] -; CHECK-SD-NEXT: add x11, sp, #312 -; CHECK-SD-NEXT: add x14, sp, #40 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x11] -; CHECK-SD-NEXT: add x11, sp, #592 -; CHECK-SD-NEXT: ld1 { v0.b }[8], [x12] -; CHECK-SD-NEXT: add x12, sp, #24 -; CHECK-SD-NEXT: add x16, sp, #248 -; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10] -; CHECK-SD-NEXT: ld1 { v4.b }[8], [x11] -; CHECK-SD-NEXT: add x11, sp, #400 -; CHECK-SD-NEXT: add x9, sp, #256 -; CHECK-SD-NEXT: add x8, sp, #264 -; CHECK-SD-NEXT: add x10, sp, #72 -; CHECK-SD-NEXT: ld1 { v0.b }[9], [x12] -; CHECK-SD-NEXT: add x12, sp, #392 -; CHECK-SD-NEXT: movi v16.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x12] -; CHECK-SD-NEXT: add x12, sp, #48 -; CHECK-SD-NEXT: ld1 { v1.b }[12], [x15] -; CHECK-SD-NEXT: add x15, sp, #120 -; CHECK-SD-NEXT: movi v17.2d, #0000000000000000 -; CHECK-SD-NEXT: movi v18.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v0.b }[10], [x13] -; CHECK-SD-NEXT: ld1 { v3.b }[5], [x15] -; CHECK-SD-NEXT: add x15, sp, #408 -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11] -; CHECK-SD-NEXT: add x13, sp, #56 -; CHECK-SD-NEXT: ld1 { v1.b }[13], [x16] -; CHECK-SD-NEXT: add x11, sp, #64 -; CHECK-SD-NEXT: add x16, sp, #616 -; CHECK-SD-NEXT: movi v19.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v0.b }[11], [x14] -; CHECK-SD-NEXT: add x14, sp, #600 -; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] -; CHECK-SD-NEXT: ld1 { v5.b }[9], [x15] -; CHECK-SD-NEXT: add x15, sp, #608 -; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9] -; CHECK-SD-NEXT: add x9, sp, #488 -; CHECK-SD-NEXT: add x14, sp, #320 -; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x14] -; CHECK-SD-NEXT: ld1 { v4.b }[10], [x15] -; CHECK-SD-NEXT: add x14, sp, #624 -; CHECK-SD-NEXT: add x9, sp, #688 -; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #432 -; CHECK-SD-NEXT: add x12, sp, #328 -; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13] -; CHECK-SD-NEXT: add x13, sp, #416 -; CHECK-SD-NEXT: ld1 { v2.b }[7], [x12] -; CHECK-SD-NEXT: ld1 { v5.b }[10], [x13] -; CHECK-SD-NEXT: ld1 { v4.b }[11], [x16] -; CHECK-SD-NEXT: add x16, sp, #680 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x16] -; CHECK-SD-NEXT: add x13, sp, #632 -; CHECK-SD-NEXT: add x12, sp, #504 -; CHECK-SD-NEXT: ld1 { v0.b }[14], [x11] -; CHECK-SD-NEXT: add x11, sp, #424 -; CHECK-SD-NEXT: add x15, sp, #128 -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x11] -; CHECK-SD-NEXT: ld1 { v4.b }[12], [x14] -; CHECK-SD-NEXT: add x11, sp, #696 -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9] -; CHECK-SD-NEXT: ld1 { v3.b }[6], [x15] -; CHECK-SD-NEXT: add x9, sp, #640 -; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] -; CHECK-SD-NEXT: add x10, sp, #496 -; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10] -; CHECK-SD-NEXT: ld1 { v4.b }[13], [x13] -; CHECK-SD-NEXT: add x10, sp, #440 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x11] -; CHECK-SD-NEXT: add x11, sp, #512 -; CHECK-SD-NEXT: add x8, sp, #136 -; CHECK-SD-NEXT: usdot v17.4s, v0.16b, v1.16b -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x10] -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x12] -; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] -; CHECK-SD-NEXT: add x9, sp, #448 -; CHECK-SD-NEXT: add x10, sp, #704 -; CHECK-SD-NEXT: ld1 { v3.b }[7], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10] -; CHECK-SD-NEXT: add x8, sp, #648 -; CHECK-SD-NEXT: add x10, sp, #520 -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9] -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11] -; CHECK-SD-NEXT: ld1 { v4.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #456 -; CHECK-SD-NEXT: add x9, sp, #712 -; CHECK-SD-NEXT: usdot v19.2s, v3.8b, v2.8b -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9] -; CHECK-SD-NEXT: addv s0, v17.4s -; CHECK-SD-NEXT: ld1 { v5.b }[15], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10] -; CHECK-SD-NEXT: addp v1.2s, v19.2s, v19.2s -; CHECK-SD-NEXT: fmov w8, s0 -; CHECK-SD-NEXT: usdot v16.4s, v5.16b, v4.16b -; CHECK-SD-NEXT: usdot v18.2s, v7.8b, v6.8b -; CHECK-SD-NEXT: fmov w9, s1 -; CHECK-SD-NEXT: addv s2, v16.4s -; CHECK-SD-NEXT: addp v3.2s, v18.2s, v18.2s -; CHECK-SD-NEXT: add w8, w8, w9 -; CHECK-SD-NEXT: fmov w10, s2 -; CHECK-SD-NEXT: fmov w11, s3 -; CHECK-SD-NEXT: add w9, w10, w11 -; CHECK-SD-NEXT: add w0, w8, w9 -; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_usdot_v24i8_double: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 -; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: ldr w9, [sp, #16] -; CHECK-GI-NEXT: ldr w8, [sp, #24] -; CHECK-GI-NEXT: fmov s0, w0 -; CHECK-GI-NEXT: ldr w11, [sp, #208] -; CHECK-GI-NEXT: ldr w13, [sp, #32] -; CHECK-GI-NEXT: ldr w12, [sp, #144] -; CHECK-GI-NEXT: fmov s1, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #80] -; CHECK-GI-NEXT: ldr w10, [sp, #152] -; CHECK-GI-NEXT: fmov s4, w11 -; CHECK-GI-NEXT: fmov s3, w12 -; CHECK-GI-NEXT: ldr w11, [sp, #160] -; CHECK-GI-NEXT: fmov s2, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #216] -; CHECK-GI-NEXT: ldr w14, [sp, #128] -; CHECK-GI-NEXT: mov v1.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #88] -; CHECK-GI-NEXT: ldr w12, [sp, #240] -; CHECK-GI-NEXT: mov v4.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #96] -; CHECK-GI-NEXT: mov v3.b[1], w10 -; CHECK-GI-NEXT: mov v2.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #40] -; CHECK-GI-NEXT: ldr w10, [sp, #272] -; CHECK-GI-NEXT: ldr w15, [sp, #312] -; CHECK-GI-NEXT: mov v0.b[1], w1 -; CHECK-GI-NEXT: mov v1.b[2], w13 -; CHECK-GI-NEXT: fmov s5, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #48] -; CHECK-GI-NEXT: mov v3.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #232] -; CHECK-GI-NEXT: ldr w13, [sp, #72] -; CHECK-GI-NEXT: mov v2.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #280] -; CHECK-GI-NEXT: mov v0.b[2], w2 -; CHECK-GI-NEXT: mov v1.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #224] -; CHECK-GI-NEXT: mov v5.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #168] -; CHECK-GI-NEXT: mov v4.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #104] -; CHECK-GI-NEXT: mov v3.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #64] -; CHECK-GI-NEXT: mov v0.b[3], w3 -; CHECK-GI-NEXT: mov v2.b[3], w8 -; CHECK-GI-NEXT: mov v1.b[4], w10 -; CHECK-GI-NEXT: ldr w8, [sp, #288] -; CHECK-GI-NEXT: ldr w10, [sp, #56] -; CHECK-GI-NEXT: mov v4.b[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #112] -; CHECK-GI-NEXT: mov v5.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #176] -; CHECK-GI-NEXT: mov v0.b[4], w4 -; CHECK-GI-NEXT: mov v2.b[4], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #120] -; CHECK-GI-NEXT: mov v1.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #296] -; CHECK-GI-NEXT: mov v3.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #136] -; CHECK-GI-NEXT: mov v4.b[4], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #248] -; CHECK-GI-NEXT: mov v5.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #184] -; CHECK-GI-NEXT: mov v0.b[5], w5 -; CHECK-GI-NEXT: mov v2.b[5], w11 -; CHECK-GI-NEXT: mov v1.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #304] -; CHECK-GI-NEXT: mov v3.b[5], w10 -; CHECK-GI-NEXT: ldr w11, [sp, #192] -; CHECK-GI-NEXT: ldr w10, [sp, #200] -; CHECK-GI-NEXT: mov v4.b[5], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #264] -; CHECK-GI-NEXT: mov v5.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #256] -; CHECK-GI-NEXT: mov v0.b[6], w6 -; CHECK-GI-NEXT: mov v2.b[6], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #336] -; CHECK-GI-NEXT: mov v1.b[7], w13 -; CHECK-GI-NEXT: mov v3.b[6], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #344] -; CHECK-GI-NEXT: ldr w13, [sp, #320] -; CHECK-GI-NEXT: fmov s6, w14 -; CHECK-GI-NEXT: ldr w14, [sp, #400] -; CHECK-GI-NEXT: mov v4.b[6], w9 -; CHECK-GI-NEXT: mov v5.b[5], w15 -; CHECK-GI-NEXT: ldr w15, [sp, #408] -; CHECK-GI-NEXT: ldr w9, [sp, #328] -; CHECK-GI-NEXT: fmov s7, w14 -; CHECK-GI-NEXT: ldr w14, [sp, #352] -; CHECK-GI-NEXT: mov v2.b[7], w8 -; CHECK-GI-NEXT: mov v6.b[1], w11 -; CHECK-GI-NEXT: mov v3.b[7], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #416] -; CHECK-GI-NEXT: mov v4.b[7], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #472] -; CHECK-GI-NEXT: ldr w11, [sp, #360] -; CHECK-GI-NEXT: mov v7.b[1], w15 -; CHECK-GI-NEXT: ldr w15, [sp, #464] -; CHECK-GI-NEXT: mov v5.b[6], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #536] -; CHECK-GI-NEXT: ldr w8, [sp, #368] -; CHECK-GI-NEXT: mov v0.b[7], w7 -; CHECK-GI-NEXT: mov v6.b[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #528] -; CHECK-GI-NEXT: fmov s16, w15 -; CHECK-GI-NEXT: ldr w15, [sp, #600] -; CHECK-GI-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: fmov s17, w14 -; CHECK-GI-NEXT: mov v7.b[2], w10 -; CHECK-GI-NEXT: ldr w14, [sp, #592] -; CHECK-GI-NEXT: mov v16.b[1], w12 -; CHECK-GI-NEXT: mov v5.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #480] -; CHECK-GI-NEXT: fmov s18, w14 -; CHECK-GI-NEXT: ldr w14, [sp, #544] -; CHECK-GI-NEXT: mov v6.b[3], w11 -; CHECK-GI-NEXT: mov v17.b[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #424] -; CHECK-GI-NEXT: ldr w11, [sp, #608] -; CHECK-GI-NEXT: ldr w10, [sp, #376] -; CHECK-GI-NEXT: ldr w12, [sp, #384] -; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: mov v7.b[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #656] -; CHECK-GI-NEXT: mov v18.b[1], w15 -; CHECK-GI-NEXT: mov v16.b[2], w9 -; CHECK-GI-NEXT: mov v6.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #616] -; CHECK-GI-NEXT: fmov s19, w13 -; CHECK-GI-NEXT: mov v17.b[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #664] -; CHECK-GI-NEXT: ldr w13, [sp, #432] -; CHECK-GI-NEXT: ushll v20.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: mov v18.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #488] -; CHECK-GI-NEXT: sshll v21.4s, v3.4h, #0 -; CHECK-GI-NEXT: mov v19.b[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #552] -; CHECK-GI-NEXT: mov v7.b[4], w13 -; CHECK-GI-NEXT: mov v16.b[3], w11 -; CHECK-GI-NEXT: ldr w13, [sp, #496] -; CHECK-GI-NEXT: ldr w11, [sp, #440] -; CHECK-GI-NEXT: mov v17.b[3], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #672] -; CHECK-GI-NEXT: mov v6.b[5], w10 -; CHECK-GI-NEXT: mov v18.b[3], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #624] -; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 -; CHECK-GI-NEXT: mov v19.b[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #560] -; CHECK-GI-NEXT: mov v7.b[5], w11 -; CHECK-GI-NEXT: mov v16.b[4], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #680] -; CHECK-GI-NEXT: ldr w11, [sp, #504] -; CHECK-GI-NEXT: mov v17.b[4], w14 -; CHECK-GI-NEXT: mov v6.b[6], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #632] -; CHECK-GI-NEXT: mov v18.b[4], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #512] -; CHECK-GI-NEXT: ldr w9, [sp, #392] -; CHECK-GI-NEXT: mov v19.b[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #568] -; CHECK-GI-NEXT: ldr w14, [sp, #584] -; CHECK-GI-NEXT: mov v16.b[5], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #688] -; CHECK-GI-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-GI-NEXT: mov v17.b[5], w13 -; CHECK-GI-NEXT: ldr w15, [sp, #448] -; CHECK-GI-NEXT: mul v20.4s, v20.4s, v21.4s -; CHECK-GI-NEXT: mov v18.b[5], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #576] -; CHECK-GI-NEXT: mul v0.4s, v0.4s, v3.4s -; CHECK-GI-NEXT: mov v19.b[4], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #640] -; CHECK-GI-NEXT: mov v6.b[7], w9 -; CHECK-GI-NEXT: mov v16.b[6], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #696] -; CHECK-GI-NEXT: mov v7.b[6], w15 -; CHECK-GI-NEXT: mov v17.b[6], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #648] -; CHECK-GI-NEXT: ushll v22.4s, v1.4h, #0 -; CHECK-GI-NEXT: mov v18.b[6], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #704] -; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: mov v19.b[5], w10 -; CHECK-GI-NEXT: sshll v21.4s, v4.4h, #0 -; CHECK-GI-NEXT: sshll2 v4.4s, v4.8h, #0 -; CHECK-GI-NEXT: ldr w8, [sp, #456] -; CHECK-GI-NEXT: ldr w13, [sp, #520] -; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 -; CHECK-GI-NEXT: mov v17.b[7], w14 -; CHECK-GI-NEXT: ldr w10, [sp, #712] -; CHECK-GI-NEXT: sshll v5.8h, v5.8b, #0 -; CHECK-GI-NEXT: mov v18.b[7], w12 -; CHECK-GI-NEXT: mul v1.4s, v1.4s, v4.4s -; CHECK-GI-NEXT: addv s4, v20.4s -; CHECK-GI-NEXT: mov v19.b[6], w11 -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0 -; CHECK-GI-NEXT: mov v7.b[7], w8 -; CHECK-GI-NEXT: mov v16.b[7], w13 -; CHECK-GI-NEXT: ushll v23.4s, v2.4h, #0 -; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-NEXT: sshll v3.4s, v5.4h, #0 -; CHECK-GI-NEXT: sshll2 v5.4s, v5.8h, #0 -; CHECK-GI-NEXT: mul v21.4s, v22.4s, v21.4s -; CHECK-GI-NEXT: fmov w8, s4 -; CHECK-GI-NEXT: mov v19.b[7], w10 -; CHECK-GI-NEXT: sshll v4.8h, v18.8b, #0 -; CHECK-GI-NEXT: fmov w9, s0 -; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0 -; CHECK-GI-NEXT: sshll v18.4s, v17.4h, #0 -; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0 -; CHECK-GI-NEXT: mul v2.4s, v2.4s, v5.4s -; CHECK-GI-NEXT: ushll v5.8h, v7.8b, #0 -; CHECK-GI-NEXT: ushll v7.8h, v16.8b, #0 -; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0 -; CHECK-GI-NEXT: mul v3.4s, v23.4s, v3.4s -; CHECK-GI-NEXT: sshll2 v22.4s, v4.8h, #0 -; CHECK-GI-NEXT: sshll v16.8h, v19.8b, #0 -; CHECK-GI-NEXT: mul v0.4s, v0.4s, v18.4s -; CHECK-GI-NEXT: addv s18, v21.4s -; CHECK-GI-NEXT: ushll2 v19.4s, v5.8h, #0 -; CHECK-GI-NEXT: ushll v20.4s, v7.4h, #0 -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: mul v6.4s, v6.4s, v17.4s -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0 -; CHECK-GI-NEXT: sshll v23.4s, v16.4h, #0 -; CHECK-GI-NEXT: fmov w9, s18 -; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0 -; CHECK-GI-NEXT: sshll v4.4s, v4.4h, #0 -; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0 -; CHECK-GI-NEXT: mul v17.4s, v19.4s, v22.4s -; CHECK-GI-NEXT: addv s3, v3.4s -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: mul v19.4s, v20.4s, v23.4s -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: addv s1, v6.4s -; CHECK-GI-NEXT: mul v4.4s, v5.4s, v4.4s -; CHECK-GI-NEXT: mul v5.4s, v7.4s, v16.4s -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: addv s3, v17.4s -; CHECK-GI-NEXT: fmov w11, s0 -; CHECK-GI-NEXT: addv s6, v19.4s -; CHECK-GI-NEXT: fmov w12, s1 -; CHECK-GI-NEXT: addv s4, v4.4s -; CHECK-GI-NEXT: addv s5, v5.4s -; CHECK-GI-NEXT: add w9, w9, w10 -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: add w11, w11, w12 -; CHECK-GI-NEXT: fmov w12, s3 -; CHECK-GI-NEXT: fmov w13, s6 -; CHECK-GI-NEXT: fmov w14, s4 -; CHECK-GI-NEXT: add w9, w9, w10 -; CHECK-GI-NEXT: add w12, w12, w13 -; CHECK-GI-NEXT: fmov w13, s5 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w10, w11, w14 -; CHECK-GI-NEXT: add w11, w12, w13 -; CHECK-GI-NEXT: add w9, w10, w11 -; CHECK-GI-NEXT: add w0, w8, w9 -; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-GI-NEXT: ret -entry: - %az = zext <24 x i8> %a to <24 x i32> - %bz = sext <24 x i8> %b to <24 x i32> - %m1 = mul nuw nsw <24 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m1) - %cz = zext <24 x i8> %c to <24 x i32> - %dz = sext <24 x i8> %d to <24 x i32> - %m2 = mul nuw nsw <24 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %m2) - %x = add i32 %r1, %r2 - ret i32 %x -} - -define i32 @test_udot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_udot_v25i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q3, q0, [x1] -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: ldp q2, q1, [x0] -; CHECK-NEXT: umull2 v4.8h, v0.16b, v1.16b -; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b -; CHECK-NEXT: umull v1.8h, v3.8b, v2.8b -; CHECK-NEXT: umull2 v2.8h, v3.16b, v2.16b -; CHECK-NEXT: ushll v3.4s, v4.4h, #0 -; CHECK-NEXT: uaddl2 v4.4s, v1.8h, v0.8h -; CHECK-NEXT: uaddl v0.4s, v1.4h, v0.4h -; CHECK-NEXT: mov v5.s[0], v3.s[0] -; CHECK-NEXT: uaddw2 v1.4s, v4.4s, v2.8h -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: uaddw v2.4s, v5.4s, v2.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret -entry: - %0 = load <25 x i8>, ptr %a - %1 = zext <25 x i8> %0 to <25 x i32> - %2 = load <25 x i8>, ptr %b - %3 = zext <25 x i8> %2 to <25 x i32> - %4 = mul nuw nsw <25 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) - %op.extra = add i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_udot_v25i8_nomla(ptr nocapture readonly %a1) { -; CHECK-LABEL: test_udot_v25i8_nomla: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q1, [x0] -; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v4.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: ushll v3.4s, v3.4h, #0 -; CHECK-NEXT: uaddl2 v5.4s, v4.8h, v1.8h -; CHECK-NEXT: uaddl v1.4s, v4.4h, v1.4h -; CHECK-NEXT: mov v0.s[0], v3.s[0] -; CHECK-NEXT: uaddw2 v3.4s, v5.4s, v2.8h -; CHECK-NEXT: add v1.4s, v1.4s, v3.4s -; CHECK-NEXT: uaddw v0.4s, v0.4s, v2.4h -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ret -entry: - %0 = load <25 x i8>, ptr %a1 - %1 = zext <25 x i8> %0 to <25 x i32> - %2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %1) - ret i32 %2 -} -define i32 @test_sdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_sdot_v25i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q3, q0, [x1] -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: ldp q2, q1, [x0] -; CHECK-NEXT: smull2 v4.8h, v0.16b, v1.16b -; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b -; CHECK-NEXT: smull v1.8h, v3.8b, v2.8b -; CHECK-NEXT: smull2 v2.8h, v3.16b, v2.16b -; CHECK-NEXT: sshll v3.4s, v4.4h, #0 -; CHECK-NEXT: saddl2 v4.4s, v1.8h, v0.8h -; CHECK-NEXT: saddl v0.4s, v1.4h, v0.4h -; CHECK-NEXT: mov v5.s[0], v3.s[0] -; CHECK-NEXT: saddw2 v1.4s, v4.4s, v2.8h -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: saddw v2.4s, v5.4s, v2.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret -entry: - %0 = load <25 x i8>, ptr %a - %1 = sext <25 x i8> %0 to <25 x i32> - %2 = load <25 x i8>, ptr %b - %3 = sext <25 x i8> %2 to <25 x i32> - %4 = mul nsw <25 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) - %op.extra = add nsw i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) { -; CHECK-LABEL: test_sdot_v25i8_double: +define i32 @test_sdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) { +; CHECK-LABEL: test_sdot_v25i8_double: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 16 @@ -3266,284 +2630,6 @@ entry: ret i32 %x } -define i32 @test_usdot_v25i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_usdot_v25i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldp q2, q0, [x0] -; CHECK-NEXT: ldp q5, q1, [x1] -; CHECK-NEXT: ushll2 v3.8h, v0.16b, #0 -; CHECK-NEXT: ushll v6.8h, v2.8b, #0 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll2 v4.8h, v1.16b, #0 -; CHECK-NEXT: sshll v7.8h, v5.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: smull v3.4s, v4.4h, v3.4h -; CHECK-NEXT: movi v4.2d, #0000000000000000 -; CHECK-NEXT: smull2 v16.4s, v7.8h, v6.8h -; CHECK-NEXT: smull v6.4s, v7.4h, v6.4h -; CHECK-NEXT: mov v4.s[0], v3.s[0] -; CHECK-NEXT: sshll2 v3.8h, v5.16b, #0 -; CHECK-NEXT: smlal2 v16.4s, v1.8h, v0.8h -; CHECK-NEXT: smlal v6.4s, v1.4h, v0.4h -; CHECK-NEXT: smlal v4.4s, v3.4h, v2.4h -; CHECK-NEXT: smlal2 v16.4s, v3.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v4.4s -; CHECK-NEXT: add v0.4s, v0.4s, v16.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret -entry: - %0 = load <25 x i8>, ptr %a - %1 = zext <25 x i8> %0 to <25 x i32> - %2 = load <25 x i8>, ptr %b - %3 = sext <25 x i8> %2 to <25 x i32> - %4 = mul nsw <25 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %4) - %op.extra = add nsw i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_usdot_v25i8_double(<25 x i8> %a, <25 x i8> %b, <25 x i8> %c, <25 x i8> %d) { -; CHECK-LABEL: test_usdot_v25i8_double: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: ldr b1, [sp, #16] -; CHECK-NEXT: ldr b0, [sp, #80] -; CHECK-NEXT: add x11, sp, #24 -; CHECK-NEXT: ldr b3, [sp, #216] -; CHECK-NEXT: ldr b4, [sp, #152] -; CHECK-NEXT: add x10, sp, #88 -; CHECK-NEXT: ldr b2, [sp, #280] -; CHECK-NEXT: ld1 { v1.b }[1], [x11] -; CHECK-NEXT: add x11, sp, #224 -; CHECK-NEXT: add x12, sp, #160 -; CHECK-NEXT: ldr b6, [sp, #480] -; CHECK-NEXT: ld1 { v0.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #288 -; CHECK-NEXT: ld1 { v3.b }[1], [x11] -; CHECK-NEXT: ld1 { v4.b }[1], [x12] -; CHECK-NEXT: add x12, sp, #488 -; CHECK-NEXT: ld1 { v2.b }[1], [x10] -; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: ld1 { v6.b }[1], [x12] -; CHECK-NEXT: add x12, sp, #32 -; CHECK-NEXT: ld1 { v0.b }[2], [x9] -; CHECK-NEXT: ld1 { v1.b }[2], [x12] -; CHECK-NEXT: add x12, sp, #232 -; CHECK-NEXT: add x9, sp, #296 -; CHECK-NEXT: ld1 { v3.b }[2], [x12] -; CHECK-NEXT: add x12, sp, #168 -; CHECK-NEXT: ld1 { v2.b }[2], [x9] -; CHECK-NEXT: ld1 { v4.b }[2], [x12] -; CHECK-NEXT: add x12, sp, #40 -; CHECK-NEXT: add x8, sp, #104 -; CHECK-NEXT: ld1 { v1.b }[3], [x12] -; CHECK-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #304 -; CHECK-NEXT: ld1 { v2.b }[3], [x8] -; CHECK-NEXT: add x8, sp, #48 -; CHECK-NEXT: add x10, sp, #112 -; CHECK-NEXT: add x14, sp, #56 -; CHECK-NEXT: add x12, sp, #240 -; CHECK-NEXT: add x11, sp, #120 -; CHECK-NEXT: ld1 { v1.b }[4], [x8] -; CHECK-NEXT: ld1 { v0.b }[4], [x10] -; CHECK-NEXT: add x8, sp, #312 -; CHECK-NEXT: ld1 { v3.b }[3], [x12] -; CHECK-NEXT: add x12, sp, #176 -; CHECK-NEXT: ld1 { v2.b }[4], [x8] -; CHECK-NEXT: ld1 { v4.b }[3], [x12] -; CHECK-NEXT: add x12, sp, #64 -; CHECK-NEXT: ldr b18, [sp, #552] -; CHECK-NEXT: ld1 { v1.b }[5], [x14] -; CHECK-NEXT: add x16, sp, #320 -; CHECK-NEXT: ld1 { v0.b }[5], [x11] -; CHECK-NEXT: ldr b16, [sp, #352] -; CHECK-NEXT: add x8, sp, #248 -; CHECK-NEXT: ld1 { v2.b }[5], [x16] -; CHECK-NEXT: add x16, sp, #360 -; CHECK-NEXT: add x15, sp, #128 -; CHECK-NEXT: ld1 { v3.b }[4], [x8] -; CHECK-NEXT: ld1 { v1.b }[6], [x12] -; CHECK-NEXT: add x12, sp, #560 -; CHECK-NEXT: ld1 { v16.b }[1], [x16] -; CHECK-NEXT: ld1 { v18.b }[1], [x12] -; CHECK-NEXT: add x10, sp, #72 -; CHECK-NEXT: ld1 { v0.b }[6], [x15] -; CHECK-NEXT: add x15, sp, #184 -; CHECK-NEXT: add x12, sp, #568 -; CHECK-NEXT: add x13, sp, #328 -; CHECK-NEXT: add x14, sp, #256 -; CHECK-NEXT: ld1 { v4.b }[4], [x15] -; CHECK-NEXT: ld1 { v1.b }[7], [x10] -; CHECK-NEXT: add x10, sp, #368 -; CHECK-NEXT: ld1 { v18.b }[2], [x12] -; CHECK-NEXT: add x9, sp, #136 -; CHECK-NEXT: ld1 { v3.b }[5], [x14] -; CHECK-NEXT: ld1 { v2.b }[6], [x13] -; CHECK-NEXT: add x13, sp, #496 -; CHECK-NEXT: ld1 { v16.b }[2], [x10] -; CHECK-NEXT: fmov s5, w0 -; CHECK-NEXT: ld1 { v0.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #192 -; CHECK-NEXT: ld1 { v6.b }[2], [x13] -; CHECK-NEXT: add x10, sp, #576 -; CHECK-NEXT: add x11, sp, #264 -; CHECK-NEXT: ld1 { v4.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #376 -; CHECK-NEXT: ld1 { v18.b }[3], [x10] -; CHECK-NEXT: ld1 { v3.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #504 -; CHECK-NEXT: ld1 { v16.b }[3], [x9] -; CHECK-NEXT: mov v5.b[1], w1 -; CHECK-NEXT: ldr b7, [sp, #144] -; CHECK-NEXT: ldr b17, [sp, #344] -; CHECK-NEXT: add x9, sp, #200 -; CHECK-NEXT: ld1 { v6.b }[3], [x11] -; CHECK-NEXT: add x10, sp, #584 -; CHECK-NEXT: ld1 { v4.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #384 -; CHECK-NEXT: ld1 { v18.b }[4], [x10] -; CHECK-NEXT: ushll v7.8h, v7.8b, #0 -; CHECK-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-NEXT: add x11, sp, #512 -; CHECK-NEXT: ld1 { v16.b }[4], [x9] -; CHECK-NEXT: mov v5.b[2], w2 -; CHECK-NEXT: ld1 { v6.b }[4], [x11] -; CHECK-NEXT: add x11, sp, #592 -; CHECK-NEXT: add x10, sp, #392 -; CHECK-NEXT: ldr b19, [sp, #680] -; CHECK-NEXT: ld1 { v18.b }[5], [x11] -; CHECK-NEXT: smull v7.4s, v7.4h, v17.4h -; CHECK-NEXT: ldr b17, [sp, #416] -; CHECK-NEXT: ld1 { v16.b }[5], [x10] -; CHECK-NEXT: add x10, sp, #688 -; CHECK-NEXT: add x9, sp, #424 -; CHECK-NEXT: ld1 { v19.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #600 -; CHECK-NEXT: ldr b20, [sp, #616] -; CHECK-NEXT: ld1 { v17.b }[1], [x9] -; CHECK-NEXT: add x11, sp, #400 -; CHECK-NEXT: ld1 { v18.b }[6], [x10] -; CHECK-NEXT: add x12, sp, #624 -; CHECK-NEXT: mov v5.b[3], w3 -; CHECK-NEXT: ld1 { v16.b }[6], [x11] -; CHECK-NEXT: add x11, sp, #696 -; CHECK-NEXT: ld1 { v20.b }[1], [x12] -; CHECK-NEXT: add x9, sp, #432 -; CHECK-NEXT: ld1 { v19.b }[2], [x11] -; CHECK-NEXT: add x11, sp, #608 -; CHECK-NEXT: ld1 { v17.b }[2], [x9] -; CHECK-NEXT: ld1 { v18.b }[7], [x11] -; CHECK-NEXT: add x11, sp, #632 -; CHECK-NEXT: add x10, sp, #408 -; CHECK-NEXT: ld1 { v20.b }[2], [x11] -; CHECK-NEXT: mov v5.b[4], w4 -; CHECK-NEXT: ld1 { v16.b }[7], [x10] -; CHECK-NEXT: add x10, sp, #704 -; CHECK-NEXT: add x12, sp, #440 -; CHECK-NEXT: ld1 { v19.b }[3], [x10] -; CHECK-NEXT: ld1 { v17.b }[3], [x12] -; CHECK-NEXT: add x12, sp, #640 -; CHECK-NEXT: ld1 { v20.b }[3], [x12] -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-NEXT: add x10, sp, #448 -; CHECK-NEXT: ushll v21.8h, v16.8b, #0 -; CHECK-NEXT: add x11, sp, #712 -; CHECK-NEXT: mov v5.b[5], w5 -; CHECK-NEXT: ld1 { v19.b }[4], [x11] -; CHECK-NEXT: ld1 { v17.b }[4], [x10] -; CHECK-NEXT: add x10, sp, #648 -; CHECK-NEXT: add x9, sp, #520 -; CHECK-NEXT: ld1 { v20.b }[4], [x10] -; CHECK-NEXT: ldr b22, [sp, #544] -; CHECK-NEXT: smull2 v16.4s, v21.8h, v18.8h -; CHECK-NEXT: smull v18.4s, v21.4h, v18.4h -; CHECK-NEXT: ldr b21, [sp, #744] -; CHECK-NEXT: add x11, sp, #720 -; CHECK-NEXT: ld1 { v6.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #456 -; CHECK-NEXT: ld1 { v19.b }[5], [x11] -; CHECK-NEXT: ld1 { v17.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #656 -; CHECK-NEXT: mov v5.b[6], w6 -; CHECK-NEXT: ushll v22.8h, v22.8b, #0 -; CHECK-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-NEXT: ld1 { v20.b }[5], [x9] -; CHECK-NEXT: add x10, sp, #528 -; CHECK-NEXT: add x11, sp, #728 -; CHECK-NEXT: ld1 { v6.b }[6], [x10] -; CHECK-NEXT: ld1 { v19.b }[6], [x11] -; CHECK-NEXT: add x10, sp, #464 -; CHECK-NEXT: add x11, sp, #664 -; CHECK-NEXT: smull v21.4s, v22.4h, v21.4h -; CHECK-NEXT: movi v22.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v17.b }[6], [x10] -; CHECK-NEXT: ld1 { v20.b }[6], [x11] -; CHECK-NEXT: mov v5.b[7], w7 -; CHECK-NEXT: add x9, sp, #536 -; CHECK-NEXT: add x10, sp, #736 -; CHECK-NEXT: add x11, sp, #208 -; CHECK-NEXT: ld1 { v6.b }[7], [x9] -; CHECK-NEXT: ld1 { v19.b }[7], [x10] -; CHECK-NEXT: ld1 { v4.b }[7], [x11] -; CHECK-NEXT: add x9, sp, #472 -; CHECK-NEXT: add x10, sp, #672 -; CHECK-NEXT: add x8, sp, #336 -; CHECK-NEXT: ld1 { v17.b }[7], [x9] -; CHECK-NEXT: ld1 { v20.b }[7], [x10] -; CHECK-NEXT: mov v22.s[0], v21.s[0] -; CHECK-NEXT: movi v21.2d, #0000000000000000 -; CHECK-NEXT: ushll v5.8h, v5.8b, #0 -; CHECK-NEXT: ushll v6.8h, v6.8b, #0 -; CHECK-NEXT: sshll v19.8h, v19.8b, #0 -; CHECK-NEXT: ld1 { v2.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #272 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: ld1 { v3.b }[7], [x8] -; CHECK-NEXT: ushll v17.8h, v17.8b, #0 -; CHECK-NEXT: sshll v20.8h, v20.8b, #0 -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: smlal v18.4s, v6.4h, v19.4h -; CHECK-NEXT: smlal2 v16.4s, v6.8h, v19.8h -; CHECK-NEXT: mov v21.s[0], v7.s[0] -; CHECK-NEXT: smull v6.4s, v5.4h, v4.4h -; CHECK-NEXT: sshll v2.8h, v2.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: smlal v22.4s, v17.4h, v20.4h -; CHECK-NEXT: smull2 v4.4s, v5.8h, v4.8h -; CHECK-NEXT: smlal v21.4s, v1.4h, v3.4h -; CHECK-NEXT: smlal2 v16.4s, v17.8h, v20.8h -; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h -; CHECK-NEXT: add v5.4s, v18.4s, v22.4s -; CHECK-NEXT: smlal2 v4.4s, v0.8h, v2.8h -; CHECK-NEXT: add v0.4s, v6.4s, v21.4s -; CHECK-NEXT: add v2.4s, v5.4s, v16.4s -; CHECK-NEXT: smlal2 v4.4s, v1.8h, v3.8h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: add v0.4s, v0.4s, v4.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: ret -entry: - %az = zext <25 x i8> %a to <25 x i32> - %bz = sext <25 x i8> %b to <25 x i32> - %m1 = mul nuw nsw <25 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m1) - %cz = zext <25 x i8> %c to <25 x i32> - %dz = sext <25 x i8> %d to <25 x i32> - %m2 = mul nuw nsw <25 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v25i32(<25 x i32> %m2) - %x = add i32 %r1, %r2 - ret i32 %x -} - define i32 @test_udot_v32i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { ; CHECK-SD-LABEL: test_udot_v32i8: ; CHECK-SD: // %bb.0: // %entry @@ -4371,2941 +3457,1507 @@ define i32 @test_sdot_v33i8_double_nomla(<33 x i8> %a, <33 x i8> %b, <33 x i8> % ; CHECK-NEXT: add x12, sp, #616 ; CHECK-NEXT: ldr b16, [sp, #544] ; CHECK-NEXT: ld1 { v0.b }[4], [x10] -; CHECK-NEXT: add x10, sp, #48 -; CHECK-NEXT: ld1 { v4.b }[1], [x12] -; CHECK-NEXT: add x12, sp, #176 -; CHECK-NEXT: ld1 { v5.b }[2], [x13] -; CHECK-NEXT: add x13, sp, #680 -; CHECK-NEXT: ld1 { v3.b }[4], [x10] -; CHECK-NEXT: ld1 { v2.b }[4], [x12] -; CHECK-NEXT: ld1 { v6.b }[1], [x13] -; CHECK-NEXT: add x13, sp, #56 -; CHECK-NEXT: ld1 { v0.b }[5], [x9] -; CHECK-NEXT: mov v1.b[2], w2 -; CHECK-NEXT: add x8, sp, #128 -; CHECK-NEXT: add x14, sp, #184 -; CHECK-NEXT: add x11, sp, #136 -; CHECK-NEXT: ld1 { v3.b }[5], [x13] -; CHECK-NEXT: add x13, sp, #552 -; CHECK-NEXT: ld1 { v2.b }[5], [x14] -; CHECK-NEXT: ld1 { v16.b }[1], [x13] -; CHECK-NEXT: add x14, sp, #624 -; CHECK-NEXT: ld1 { v0.b }[6], [x8] -; CHECK-NEXT: add x8, sp, #688 -; CHECK-NEXT: add x13, sp, #504 -; CHECK-NEXT: ld1 { v4.b }[2], [x14] -; CHECK-NEXT: ld1 { v6.b }[2], [x8] -; CHECK-NEXT: add x8, sp, #560 -; CHECK-NEXT: ld1 { v5.b }[3], [x13] -; CHECK-NEXT: ld1 { v16.b }[2], [x8] -; CHECK-NEXT: mov v1.b[3], w3 -; CHECK-NEXT: add x9, sp, #64 -; CHECK-NEXT: add x15, sp, #632 -; CHECK-NEXT: ld1 { v3.b }[6], [x9] -; CHECK-NEXT: ld1 { v0.b }[7], [x11] -; CHECK-NEXT: ld1 { v4.b }[3], [x15] -; CHECK-NEXT: add x8, sp, #696 -; CHECK-NEXT: add x9, sp, #568 -; CHECK-NEXT: add x11, sp, #512 -; CHECK-NEXT: ld1 { v6.b }[3], [x8] -; CHECK-NEXT: ld1 { v16.b }[3], [x9] -; CHECK-NEXT: ld1 { v5.b }[4], [x11] -; CHECK-NEXT: add x8, sp, #640 -; CHECK-NEXT: mov v1.b[4], w4 -; CHECK-NEXT: ld1 { v4.b }[4], [x8] -; CHECK-NEXT: add x8, sp, #704 -; CHECK-NEXT: add x9, sp, #576 -; CHECK-NEXT: add x11, sp, #520 -; CHECK-NEXT: ld1 { v6.b }[4], [x8] -; CHECK-NEXT: ld1 { v16.b }[4], [x9] -; CHECK-NEXT: ld1 { v5.b }[5], [x11] -; CHECK-NEXT: ldr b18, [sp, #736] -; CHECK-NEXT: add x12, sp, #192 -; CHECK-NEXT: ld1 { v2.b }[6], [x12] -; CHECK-NEXT: add x8, sp, #648 -; CHECK-NEXT: add x9, sp, #528 -; CHECK-NEXT: add x11, sp, #712 -; CHECK-NEXT: add x12, sp, #584 -; CHECK-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-NEXT: mov v1.b[5], w5 -; CHECK-NEXT: ld1 { v6.b }[5], [x11] -; CHECK-NEXT: ld1 { v16.b }[5], [x12] -; CHECK-NEXT: ld1 { v4.b }[5], [x8] -; CHECK-NEXT: ld1 { v5.b }[6], [x9] -; CHECK-NEXT: movi v17.2d, #0000000000000000 -; CHECK-NEXT: add x8, sp, #656 -; CHECK-NEXT: add x9, sp, #536 -; CHECK-NEXT: add x11, sp, #720 -; CHECK-NEXT: add x12, sp, #592 -; CHECK-NEXT: sshll v18.4s, v18.4h, #0 -; CHECK-NEXT: ldr b7, [sp, #208] -; CHECK-NEXT: ld1 { v6.b }[6], [x11] -; CHECK-NEXT: ld1 { v16.b }[6], [x12] -; CHECK-NEXT: ld1 { v4.b }[6], [x8] -; CHECK-NEXT: ld1 { v5.b }[7], [x9] -; CHECK-NEXT: mov v1.b[6], w6 -; CHECK-NEXT: sshll v7.8h, v7.8b, #0 -; CHECK-NEXT: add x8, sp, #664 -; CHECK-NEXT: add x9, sp, #728 -; CHECK-NEXT: add x11, sp, #600 -; CHECK-NEXT: mov v17.s[0], v18.s[0] -; CHECK-NEXT: ld1 { v6.b }[7], [x9] -; CHECK-NEXT: ld1 { v16.b }[7], [x11] -; CHECK-NEXT: ld1 { v4.b }[7], [x8] -; CHECK-NEXT: sshll v5.8h, v5.8b, #0 -; CHECK-NEXT: movi v18.2d, #0000000000000000 -; CHECK-NEXT: add x10, sp, #200 -; CHECK-NEXT: mov v1.b[7], w7 -; CHECK-NEXT: add x9, sp, #72 -; CHECK-NEXT: sshll v7.4s, v7.4h, #0 -; CHECK-NEXT: ld1 { v2.b }[7], [x10] -; CHECK-NEXT: ld1 { v3.b }[7], [x9] -; CHECK-NEXT: sshll v6.8h, v6.8b, #0 -; CHECK-NEXT: sshll v16.8h, v16.8b, #0 -; CHECK-NEXT: sshll v4.8h, v4.8b, #0 -; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h -; CHECK-NEXT: sshll v0.8h, v0.8b, #0 -; CHECK-NEXT: mov v18.s[0], v7.s[0] -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v2.8h, v2.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: saddl2 v7.4s, v16.8h, v6.8h -; CHECK-NEXT: saddl2 v5.4s, v5.8h, v4.8h -; CHECK-NEXT: saddl v6.4s, v16.4h, v6.4h -; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h -; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h -; CHECK-NEXT: saddl2 v16.4s, v3.8h, v2.8h -; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h -; CHECK-NEXT: add v5.4s, v5.4s, v7.4s -; CHECK-NEXT: add v4.4s, v4.4s, v6.4s -; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h -; CHECK-NEXT: add v6.4s, v17.4s, v16.4s -; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h -; CHECK-NEXT: add v1.4s, v4.4s, v5.4s -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: add v1.4s, v6.4s, v1.4s -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: ret -entry: - %az = sext <33 x i8> %a to <33 x i32> - %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az) - %cz = sext <33 x i8> %c to <33 x i32> - %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz) - %x = add i32 %r1, %r2 - ret i32 %x -} - -define i32 @test_usdot_v33i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-LABEL: test_usdot_v33i8: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: ldr b0, [x0, #32] -; CHECK-NEXT: ldr b1, [x1, #32] -; CHECK-NEXT: ldp q2, q4, [x0] -; CHECK-NEXT: ldp q3, q6, [x1] -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-NEXT: ushll v5.8h, v2.8b, #0 -; CHECK-NEXT: ushll2 v2.8h, v2.16b, #0 -; CHECK-NEXT: ushll v16.8h, v4.8b, #0 -; CHECK-NEXT: sshll v7.8h, v3.8b, #0 -; CHECK-NEXT: sshll2 v3.8h, v3.16b, #0 -; CHECK-NEXT: ushll2 v4.8h, v4.16b, #0 -; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: sshll v19.8h, v6.8b, #0 -; CHECK-NEXT: sshll2 v6.8h, v6.16b, #0 -; CHECK-NEXT: smull2 v17.4s, v7.8h, v5.8h -; CHECK-NEXT: smull2 v18.4s, v3.8h, v2.8h -; CHECK-NEXT: mov v1.s[0], v0.s[0] -; CHECK-NEXT: smull v0.4s, v3.4h, v2.4h -; CHECK-NEXT: smlal2 v18.4s, v6.8h, v4.8h -; CHECK-NEXT: smlal2 v17.4s, v19.8h, v16.8h -; CHECK-NEXT: smlal v1.4s, v7.4h, v5.4h -; CHECK-NEXT: smlal v0.4s, v6.4h, v4.4h -; CHECK-NEXT: add v2.4s, v17.4s, v18.4s -; CHECK-NEXT: smlal v1.4s, v19.4h, v16.4h -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w8, s0 -; CHECK-NEXT: add w0, w8, w2 -; CHECK-NEXT: ret -entry: - %0 = load <33 x i8>, ptr %a - %1 = zext <33 x i8> %0 to <33 x i32> - %2 = load <33 x i8>, ptr %b - %3 = sext <33 x i8> %2 to <33 x i32> - %4 = mul nsw <33 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %4) - %op.extra = add nsw i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_usdot_v33i8_double(<33 x i8> %a, <33 x i8> %b, <33 x i8> %c, <33 x i8> %d) { -; CHECK-LABEL: test_usdot_v33i8_double: -; CHECK: // %bb.0: // %entry -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: fmov s4, w0 -; CHECK-NEXT: ldr b0, [sp, #144] -; CHECK-NEXT: add x8, sp, #152 -; CHECK-NEXT: ldr b1, [sp, #80] -; CHECK-NEXT: add x10, sp, #88 -; CHECK-NEXT: ldr b6, [sp, #16] -; CHECK-NEXT: ld1 { v0.b }[1], [x8] -; CHECK-NEXT: ldr b2, [sp, #408] -; CHECK-NEXT: add x9, sp, #160 -; CHECK-NEXT: mov v4.b[1], w1 -; CHECK-NEXT: ld1 { v1.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #24 -; CHECK-NEXT: ld1 { v6.b }[1], [x10] -; CHECK-NEXT: add x10, sp, #416 -; CHECK-NEXT: add x8, sp, #168 -; CHECK-NEXT: ld1 { v0.b }[2], [x9] -; CHECK-NEXT: add x9, sp, #96 -; CHECK-NEXT: ld1 { v2.b }[1], [x10] -; CHECK-NEXT: ld1 { v1.b }[2], [x9] -; CHECK-NEXT: add x9, sp, #32 -; CHECK-NEXT: add x10, sp, #424 -; CHECK-NEXT: mov v4.b[2], w2 -; CHECK-NEXT: ld1 { v6.b }[2], [x9] -; CHECK-NEXT: add x12, sp, #176 -; CHECK-NEXT: ld1 { v2.b }[2], [x10] -; CHECK-NEXT: add x10, sp, #104 -; CHECK-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-NEXT: ld1 { v1.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #40 -; CHECK-NEXT: add x13, sp, #112 -; CHECK-NEXT: ld1 { v6.b }[3], [x10] -; CHECK-NEXT: add x10, sp, #432 -; CHECK-NEXT: add x11, sp, #184 -; CHECK-NEXT: mov v4.b[3], w3 -; CHECK-NEXT: ld1 { v0.b }[4], [x12] -; CHECK-NEXT: add x12, sp, #48 -; CHECK-NEXT: ld1 { v2.b }[3], [x10] -; CHECK-NEXT: ld1 { v1.b }[4], [x13] -; CHECK-NEXT: add x14, sp, #440 -; CHECK-NEXT: ld1 { v6.b }[4], [x12] -; CHECK-NEXT: ldr b5, [sp, #216] -; CHECK-NEXT: add x10, sp, #120 -; CHECK-NEXT: ld1 { v0.b }[5], [x11] -; CHECK-NEXT: add x11, sp, #56 -; CHECK-NEXT: ldr b7, [sp, #280] -; CHECK-NEXT: mov v4.b[4], w4 -; CHECK-NEXT: ld1 { v2.b }[4], [x14] -; CHECK-NEXT: add x14, sp, #224 -; CHECK-NEXT: ld1 { v6.b }[5], [x11] -; CHECK-NEXT: ld1 { v5.b }[1], [x14] -; CHECK-NEXT: add x15, sp, #288 -; CHECK-NEXT: ld1 { v1.b }[5], [x10] -; CHECK-NEXT: add x14, sp, #64 -; CHECK-NEXT: ld1 { v7.b }[1], [x15] -; CHECK-NEXT: add x9, sp, #192 -; CHECK-NEXT: add x12, sp, #128 -; CHECK-NEXT: ldr b3, [sp, #344] -; CHECK-NEXT: mov v4.b[5], w5 -; CHECK-NEXT: ld1 { v6.b }[6], [x14] -; CHECK-NEXT: add x14, sp, #232 -; CHECK-NEXT: ld1 { v0.b }[6], [x9] -; CHECK-NEXT: ld1 { v5.b }[2], [x14] -; CHECK-NEXT: ld1 { v1.b }[6], [x12] -; CHECK-NEXT: add x12, sp, #296 -; CHECK-NEXT: add x13, sp, #352 -; CHECK-NEXT: add x8, sp, #200 -; CHECK-NEXT: ld1 { v7.b }[2], [x12] -; CHECK-NEXT: ld1 { v3.b }[1], [x13] -; CHECK-NEXT: add x14, sp, #240 -; CHECK-NEXT: mov v4.b[6], w6 -; CHECK-NEXT: add x10, sp, #448 -; CHECK-NEXT: ld1 { v5.b }[3], [x14] -; CHECK-NEXT: ld1 { v0.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #304 -; CHECK-NEXT: add x9, sp, #360 -; CHECK-NEXT: ld1 { v2.b }[5], [x10] -; CHECK-NEXT: ld1 { v7.b }[3], [x8] -; CHECK-NEXT: add x11, sp, #72 -; CHECK-NEXT: ld1 { v3.b }[2], [x9] -; CHECK-NEXT: add x8, sp, #248 -; CHECK-NEXT: add x15, sp, #456 -; CHECK-NEXT: mov v4.b[7], w7 -; CHECK-NEXT: ld1 { v6.b }[7], [x11] -; CHECK-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-NEXT: add x8, sp, #312 -; CHECK-NEXT: ld1 { v2.b }[6], [x15] -; CHECK-NEXT: add x9, sp, #368 -; CHECK-NEXT: ld1 { v7.b }[4], [x8] -; CHECK-NEXT: ld1 { v3.b }[3], [x9] -; CHECK-NEXT: add x8, sp, #256 -; CHECK-NEXT: ushll v17.8h, v6.8b, #0 -; CHECK-NEXT: add x10, sp, #464 -; CHECK-NEXT: ld1 { v5.b }[5], [x8] -; CHECK-NEXT: ushll v21.8h, v4.8b, #0 -; CHECK-NEXT: ldr b4, [sp, #208] -; CHECK-NEXT: add x8, sp, #320 -; CHECK-NEXT: ld1 { v2.b }[7], [x10] -; CHECK-NEXT: add x9, sp, #376 -; CHECK-NEXT: ld1 { v7.b }[5], [x8] -; CHECK-NEXT: ushll v6.8h, v4.8b, #0 -; CHECK-NEXT: ldr b4, [sp, #672] -; CHECK-NEXT: add x10, sp, #680 -; CHECK-NEXT: ld1 { v3.b }[4], [x9] -; CHECK-NEXT: add x9, sp, #264 -; CHECK-NEXT: ldr b16, [sp, #472] -; CHECK-NEXT: ld1 { v4.b }[1], [x10] -; CHECK-NEXT: ld1 { v5.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #328 -; CHECK-NEXT: ld1 { v7.b }[6], [x9] -; CHECK-NEXT: add x10, sp, #688 -; CHECK-NEXT: add x9, sp, #272 -; CHECK-NEXT: add x8, sp, #384 -; CHECK-NEXT: sshll v16.8h, v16.8b, #0 -; CHECK-NEXT: ldr b19, [sp, #480] -; CHECK-NEXT: ld1 { v4.b }[2], [x10] -; CHECK-NEXT: ld1 { v5.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #336 -; CHECK-NEXT: ld1 { v3.b }[5], [x8] -; CHECK-NEXT: ld1 { v7.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #696 -; CHECK-NEXT: add x8, sp, #392 -; CHECK-NEXT: smull v20.4s, v6.4h, v16.4h -; CHECK-NEXT: ldr b16, [sp, #608] -; CHECK-NEXT: ld1 { v4.b }[3], [x9] -; CHECK-NEXT: add x9, sp, #488 -; CHECK-NEXT: sshll v22.8h, v5.8b, #0 -; CHECK-NEXT: ld1 { v19.b }[1], [x9] -; CHECK-NEXT: ld1 { v3.b }[6], [x8] -; CHECK-NEXT: add x9, sp, #704 -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: add x10, sp, #496 -; CHECK-NEXT: add x8, sp, #400 -; CHECK-NEXT: ld1 { v4.b }[4], [x9] -; CHECK-NEXT: add x9, sp, #616 -; CHECK-NEXT: sshll v18.8h, v7.8b, #0 -; CHECK-NEXT: ld1 { v16.b }[1], [x9] -; CHECK-NEXT: ld1 { v19.b }[2], [x10] -; CHECK-NEXT: ld1 { v3.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #712 -; CHECK-NEXT: add x9, sp, #504 -; CHECK-NEXT: smull2 v7.4s, v21.8h, v22.8h -; CHECK-NEXT: ld1 { v4.b }[5], [x8] -; CHECK-NEXT: add x8, sp, #624 -; CHECK-NEXT: mov v5.s[0], v20.s[0] -; CHECK-NEXT: ld1 { v16.b }[2], [x8] -; CHECK-NEXT: ld1 { v19.b }[3], [x9] -; CHECK-NEXT: add x8, sp, #720 -; CHECK-NEXT: add x9, sp, #512 -; CHECK-NEXT: ldr b20, [sp, #544] -; CHECK-NEXT: add x10, sp, #640 -; CHECK-NEXT: ld1 { v4.b }[6], [x8] -; CHECK-NEXT: add x8, sp, #632 -; CHECK-NEXT: add x11, sp, #520 -; CHECK-NEXT: ld1 { v16.b }[3], [x8] -; CHECK-NEXT: ld1 { v19.b }[4], [x9] -; CHECK-NEXT: add x9, sp, #552 -; CHECK-NEXT: smlal v5.4s, v21.4h, v22.4h -; CHECK-NEXT: ld1 { v20.b }[1], [x9] -; CHECK-NEXT: ldr b21, [sp, #736] -; CHECK-NEXT: ldr b22, [sp, #1000] -; CHECK-NEXT: add x9, sp, #528 -; CHECK-NEXT: ldr b23, [sp, #808] -; CHECK-NEXT: ld1 { v16.b }[4], [x10] -; CHECK-NEXT: ld1 { v19.b }[5], [x11] -; CHECK-NEXT: add x10, sp, #560 -; CHECK-NEXT: ushll v21.8h, v21.8b, #0 -; CHECK-NEXT: sshll v24.8h, v22.8b, #0 -; CHECK-NEXT: ld1 { v20.b }[2], [x10] -; CHECK-NEXT: smull v6.4s, v17.4h, v18.4h -; CHECK-NEXT: smull2 v18.4s, v17.8h, v18.8h -; CHECK-NEXT: movi v17.2d, #0000000000000000 -; CHECK-NEXT: ld1 { v19.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #568 -; CHECK-NEXT: ldr b22, [sp, #744] -; CHECK-NEXT: add x11, sp, #816 -; CHECK-NEXT: smull v24.4s, v21.4h, v24.4h -; CHECK-NEXT: ld1 { v20.b }[3], [x9] -; CHECK-NEXT: add x10, sp, #752 -; CHECK-NEXT: ld1 { v23.b }[1], [x11] -; CHECK-NEXT: add x8, sp, #728 -; CHECK-NEXT: ld1 { v22.b }[1], [x10] -; CHECK-NEXT: add x9, sp, #576 -; CHECK-NEXT: ldr b21, [sp, #936] -; CHECK-NEXT: add x10, sp, #824 -; CHECK-NEXT: ld1 { v4.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #648 -; CHECK-NEXT: ld1 { v20.b }[4], [x9] -; CHECK-NEXT: add x9, sp, #760 -; CHECK-NEXT: add x11, sp, #944 -; CHECK-NEXT: ld1 { v23.b }[2], [x10] -; CHECK-NEXT: mov v17.s[0], v24.s[0] -; CHECK-NEXT: ldr b24, [sp, #872] -; CHECK-NEXT: ld1 { v16.b }[5], [x8] -; CHECK-NEXT: add x8, sp, #536 -; CHECK-NEXT: ld1 { v22.b }[2], [x9] -; CHECK-NEXT: ld1 { v21.b }[1], [x11] -; CHECK-NEXT: add x10, sp, #880 -; CHECK-NEXT: ld1 { v19.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #584 -; CHECK-NEXT: add x9, sp, #832 -; CHECK-NEXT: ld1 { v24.b }[1], [x10] -; CHECK-NEXT: ld1 { v20.b }[5], [x8] -; CHECK-NEXT: add x8, sp, #768 -; CHECK-NEXT: ld1 { v23.b }[3], [x9] -; CHECK-NEXT: add x9, sp, #952 -; CHECK-NEXT: ld1 { v22.b }[3], [x8] -; CHECK-NEXT: add x11, sp, #888 -; CHECK-NEXT: ld1 { v21.b }[2], [x9] -; CHECK-NEXT: add x8, sp, #592 -; CHECK-NEXT: add x10, sp, #840 -; CHECK-NEXT: ld1 { v24.b }[2], [x11] -; CHECK-NEXT: add x9, sp, #776 -; CHECK-NEXT: ld1 { v23.b }[4], [x10] -; CHECK-NEXT: ld1 { v20.b }[6], [x8] -; CHECK-NEXT: add x8, sp, #960 -; CHECK-NEXT: ld1 { v22.b }[4], [x9] -; CHECK-NEXT: ld1 { v21.b }[3], [x8] -; CHECK-NEXT: add x10, sp, #896 -; CHECK-NEXT: add x9, sp, #848 -; CHECK-NEXT: ld1 { v24.b }[3], [x10] -; CHECK-NEXT: add x8, sp, #784 -; CHECK-NEXT: ld1 { v23.b }[5], [x9] -; CHECK-NEXT: add x9, sp, #968 -; CHECK-NEXT: ld1 { v22.b }[5], [x8] -; CHECK-NEXT: add x11, sp, #904 -; CHECK-NEXT: ld1 { v21.b }[4], [x9] -; CHECK-NEXT: add x8, sp, #600 -; CHECK-NEXT: add x10, sp, #856 -; CHECK-NEXT: ld1 { v24.b }[4], [x11] -; CHECK-NEXT: add x9, sp, #792 -; CHECK-NEXT: ld1 { v23.b }[6], [x10] -; CHECK-NEXT: ld1 { v20.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #976 -; CHECK-NEXT: ld1 { v22.b }[6], [x9] -; CHECK-NEXT: ld1 { v21.b }[5], [x8] -; CHECK-NEXT: add x10, sp, #912 -; CHECK-NEXT: add x9, sp, #864 -; CHECK-NEXT: ld1 { v24.b }[5], [x10] -; CHECK-NEXT: add x8, sp, #800 -; CHECK-NEXT: ld1 { v23.b }[7], [x9] -; CHECK-NEXT: add x9, sp, #984 -; CHECK-NEXT: ld1 { v22.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #656 -; CHECK-NEXT: ld1 { v21.b }[6], [x9] -; CHECK-NEXT: add x9, sp, #920 -; CHECK-NEXT: ld1 { v16.b }[6], [x8] -; CHECK-NEXT: ld1 { v24.b }[6], [x9] -; CHECK-NEXT: add x8, sp, #992 -; CHECK-NEXT: add x13, sp, #136 -; CHECK-NEXT: ushll v19.8h, v19.8b, #0 -; CHECK-NEXT: ushll v20.8h, v20.8b, #0 -; CHECK-NEXT: sshll v22.8h, v22.8b, #0 -; CHECK-NEXT: sshll v23.8h, v23.8b, #0 -; CHECK-NEXT: add x9, sp, #664 -; CHECK-NEXT: ld1 { v21.b }[7], [x8] -; CHECK-NEXT: add x8, sp, #928 -; CHECK-NEXT: ld1 { v1.b }[7], [x13] -; CHECK-NEXT: ld1 { v16.b }[7], [x9] -; CHECK-NEXT: ld1 { v24.b }[7], [x8] -; CHECK-NEXT: smlal v17.4s, v19.4h, v22.4h -; CHECK-NEXT: smull2 v19.4s, v19.8h, v22.8h -; CHECK-NEXT: smull v22.4s, v20.4h, v23.4h -; CHECK-NEXT: smull2 v20.4s, v20.8h, v23.8h -; CHECK-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-NEXT: ushll v1.8h, v1.8b, #0 -; CHECK-NEXT: sshll v3.8h, v3.8b, #0 -; CHECK-NEXT: sshll v2.8h, v2.8b, #0 -; CHECK-NEXT: ushll v4.8h, v4.8b, #0 -; CHECK-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-NEXT: ushll v16.8h, v16.8b, #0 -; CHECK-NEXT: sshll v23.8h, v24.8b, #0 -; CHECK-NEXT: smlal2 v7.4s, v1.8h, v3.8h -; CHECK-NEXT: smlal v5.4s, v1.4h, v3.4h -; CHECK-NEXT: smlal2 v18.4s, v0.8h, v2.8h -; CHECK-NEXT: smlal v6.4s, v0.4h, v2.4h -; CHECK-NEXT: smlal2 v20.4s, v4.8h, v21.8h -; CHECK-NEXT: smlal v22.4s, v4.4h, v21.4h -; CHECK-NEXT: smlal2 v19.4s, v16.8h, v23.8h -; CHECK-NEXT: smlal v17.4s, v16.4h, v23.4h -; CHECK-NEXT: add v0.4s, v7.4s, v18.4s -; CHECK-NEXT: add v1.4s, v5.4s, v6.4s -; CHECK-NEXT: add v2.4s, v19.4s, v20.4s -; CHECK-NEXT: add v3.4s, v17.4s, v22.4s -; CHECK-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-NEXT: add v1.4s, v3.4s, v2.4s -; CHECK-NEXT: add v0.4s, v0.4s, v1.4s -; CHECK-NEXT: addv s0, v0.4s -; CHECK-NEXT: fmov w0, s0 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-NEXT: ret -entry: - %az = zext <33 x i8> %a to <33 x i32> - %bz = sext <33 x i8> %b to <33 x i32> - %m1 = mul nuw nsw <33 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m1) - %cz = zext <33 x i8> %c to <33 x i32> - %dz = sext <33 x i8> %d to <33 x i32> - %m2 = mul nuw nsw <33 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %m2) - %x = add i32 %r1, %r2 - ret i32 %x -} - -define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-SD-LABEL: test_udot_v48i8: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 -; CHECK-SD-NEXT: ldr q1, [x0, #32] -; CHECK-SD-NEXT: ldr q2, [x1, #32] -; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b -; CHECK-SD-NEXT: ldp q3, q1, [x0] -; CHECK-SD-NEXT: ldp q4, q2, [x1] -; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b -; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b -; CHECK-SD-NEXT: addv s0, v0.4s -; CHECK-SD-NEXT: fmov w8, s0 -; CHECK-SD-NEXT: add w0, w8, w2 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_udot_v48i8: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: movi v0.2d, #0000000000000000 -; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 -; CHECK-GI-NEXT: ldr q7, [x0, #32] -; CHECK-GI-NEXT: ldp q3, q4, [x0] -; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 -; CHECK-GI-NEXT: ldp q5, q6, [x1] -; CHECK-GI-NEXT: ldr q16, [x1, #32] -; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b -; CHECK-GI-NEXT: udot v1.4s, v6.16b, v4.16b -; CHECK-GI-NEXT: udot v2.4s, v16.16b, v7.16b -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ret -entry: - %0 = load <48 x i8>, ptr %a - %1 = zext <48 x i8> %0 to <48 x i32> - %2 = load <48 x i8>, ptr %b - %3 = zext <48 x i8> %2 to <48 x i32> - %4 = mul nuw nsw <48 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4) - %op.extra = add i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) { -; CHECK-SD-LABEL: test_udot_v48i8_nomla: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: movi v0.16b, #1 -; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 -; CHECK-SD-NEXT: ldr q2, [x0, #32] -; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b -; CHECK-SD-NEXT: ldp q3, q2, [x0] -; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b -; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b -; CHECK-SD-NEXT: addv s0, v1.4s -; CHECK-SD-NEXT: fmov w0, s0 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_udot_v48i8_nomla: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: movi v0.16b, #1 -; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 -; CHECK-GI-NEXT: ldr q6, [x0, #32] -; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 -; CHECK-GI-NEXT: ldp q4, q5, [x0] -; CHECK-GI-NEXT: movi v3.2d, #0000000000000000 -; CHECK-GI-NEXT: udot v1.4s, v4.16b, v0.16b -; CHECK-GI-NEXT: udot v2.4s, v5.16b, v0.16b -; CHECK-GI-NEXT: udot v3.4s, v6.16b, v0.16b -; CHECK-GI-NEXT: addv s0, v1.4s -; CHECK-GI-NEXT: addv s1, v2.4s -; CHECK-GI-NEXT: addv s2, v3.4s -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: add w0, w8, w9 -; CHECK-GI-NEXT: ret -entry: - %0 = load <48 x i8>, ptr %a1 - %1 = zext <48 x i8> %0 to <48 x i32> - %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1) - ret i32 %2 -} -define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-SD-LABEL: test_sdot_v48i8: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 -; CHECK-SD-NEXT: ldr q1, [x0, #32] -; CHECK-SD-NEXT: ldr q2, [x1, #32] -; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b -; CHECK-SD-NEXT: ldp q3, q1, [x0] -; CHECK-SD-NEXT: ldp q4, q2, [x1] -; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v3.16b -; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b -; CHECK-SD-NEXT: addv s0, v0.4s -; CHECK-SD-NEXT: fmov w8, s0 -; CHECK-SD-NEXT: add w0, w8, w2 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_sdot_v48i8: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: movi v0.2d, #0000000000000000 -; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 -; CHECK-GI-NEXT: ldr q7, [x0, #32] -; CHECK-GI-NEXT: ldp q3, q4, [x0] -; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 -; CHECK-GI-NEXT: ldp q5, q6, [x1] -; CHECK-GI-NEXT: ldr q16, [x1, #32] -; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b -; CHECK-GI-NEXT: sdot v1.4s, v6.16b, v4.16b -; CHECK-GI-NEXT: sdot v2.4s, v16.16b, v7.16b -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w0, w8, w2 -; CHECK-GI-NEXT: ret -entry: - %0 = load <48 x i8>, ptr %a - %1 = sext <48 x i8> %0 to <48 x i32> - %2 = load <48 x i8>, ptr %b - %3 = sext <48 x i8> %2 to <48 x i32> - %4 = mul nsw <48 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4) - %op.extra = add nsw i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { -; CHECK-SD-LABEL: test_sdot_v48i8_double: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 -; CHECK-SD-NEXT: .cfi_offset w29, -16 -; CHECK-SD-NEXT: ldr b3, [sp, #592] -; CHECK-SD-NEXT: add x8, sp, #600 -; CHECK-SD-NEXT: ldr b6, [sp, #208] -; CHECK-SD-NEXT: ldr b0, [sp, #336] -; CHECK-SD-NEXT: add x9, sp, #344 -; CHECK-SD-NEXT: ldr b2, [sp, #464] -; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #216 -; CHECK-SD-NEXT: add x10, sp, #624 -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #608 -; CHECK-SD-NEXT: ld1 { v0.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #232 -; CHECK-SD-NEXT: fmov s1, w0 -; CHECK-SD-NEXT: ldr b7, [sp, #1360] -; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #224 -; CHECK-SD-NEXT: add x11, sp, #648 -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #616 -; CHECK-SD-NEXT: add x12, sp, #376 -; CHECK-SD-NEXT: mov v1.b[1], w1 -; CHECK-SD-NEXT: ldr b16, [sp, #976] -; CHECK-SD-NEXT: add x14, sp, #288 -; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #632 -; CHECK-SD-NEXT: add x15, sp, #408 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #472 -; CHECK-SD-NEXT: add x13, sp, #696 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #240 -; CHECK-SD-NEXT: add x16, sp, #448 -; CHECK-SD-NEXT: ld1 { v3.b }[4], [x10] -; CHECK-SD-NEXT: add x10, sp, #352 -; CHECK-SD-NEXT: mov v1.b[2], w2 -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9] -; CHECK-SD-NEXT: ld1 { v0.b }[2], [x10] -; CHECK-SD-NEXT: add x10, sp, #1368 -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x10] -; CHECK-SD-NEXT: add x10, sp, #248 -; CHECK-SD-NEXT: add x9, sp, #640 -; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #656 -; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10] -; CHECK-SD-NEXT: add x10, sp, #360 -; CHECK-SD-NEXT: mov v1.b[3], w3 -; CHECK-SD-NEXT: ld1 { v0.b }[3], [x10] -; CHECK-SD-NEXT: add x10, sp, #256 -; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9] -; CHECK-SD-NEXT: add x9, sp, #368 -; CHECK-SD-NEXT: ldr b17, [sp, #720] -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10] -; CHECK-SD-NEXT: add x10, sp, #984 -; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10] -; CHECK-SD-NEXT: add x10, sp, #664 -; CHECK-SD-NEXT: ld1 { v3.b }[7], [x11] -; CHECK-SD-NEXT: add x11, sp, #264 -; CHECK-SD-NEXT: mov v1.b[4], w4 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x11] -; CHECK-SD-NEXT: add x9, sp, #672 -; CHECK-SD-NEXT: add x11, sp, #680 -; CHECK-SD-NEXT: ld1 { v0.b }[5], [x12] -; CHECK-SD-NEXT: add x12, sp, #480 -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12] -; CHECK-SD-NEXT: add x12, sp, #272 -; CHECK-SD-NEXT: ld1 { v3.b }[8], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x12] -; CHECK-SD-NEXT: add x12, sp, #384 -; CHECK-SD-NEXT: mov v1.b[5], w5 -; CHECK-SD-NEXT: ld1 { v0.b }[6], [x12] -; CHECK-SD-NEXT: add x12, sp, #280 -; CHECK-SD-NEXT: add x8, sp, #688 -; CHECK-SD-NEXT: ld1 { v3.b }[9], [x10] -; CHECK-SD-NEXT: add x10, sp, #1376 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] -; CHECK-SD-NEXT: add x10, sp, #392 -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12] -; CHECK-SD-NEXT: ld1 { v0.b }[7], [x10] -; CHECK-SD-NEXT: mov v1.b[6], w6 -; CHECK-SD-NEXT: add x12, sp, #704 -; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9] -; CHECK-SD-NEXT: add x9, sp, #400 -; CHECK-SD-NEXT: add x10, sp, #712 -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x14] -; CHECK-SD-NEXT: add x14, sp, #992 -; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[2], [x14] -; CHECK-SD-NEXT: add x14, sp, #296 -; CHECK-SD-NEXT: ld1 { v3.b }[11], [x11] -; CHECK-SD-NEXT: add x9, sp, #304 -; CHECK-SD-NEXT: add x11, sp, #312 -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x14] -; CHECK-SD-NEXT: mov v1.b[7], w7 -; CHECK-SD-NEXT: add x14, sp, #320 -; CHECK-SD-NEXT: ld1 { v0.b }[9], [x15] -; CHECK-SD-NEXT: add x15, sp, #328 -; CHECK-SD-NEXT: ld1 { v3.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #416 -; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9] -; CHECK-SD-NEXT: add x9, sp, #1384 -; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #424 -; CHECK-SD-NEXT: ld1 { v3.b }[13], [x13] -; CHECK-SD-NEXT: add x8, sp, #432 -; CHECK-SD-NEXT: add x13, sp, #440 -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11] -; CHECK-SD-NEXT: add x11, sp, #16 -; CHECK-SD-NEXT: ld1 { v0.b }[11], [x9] -; CHECK-SD-NEXT: add x9, sp, #1000 -; CHECK-SD-NEXT: ld1 { v1.b }[8], [x11] -; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9] -; CHECK-SD-NEXT: ld1 { v3.b }[14], [x12] -; CHECK-SD-NEXT: add x12, sp, #488 -; CHECK-SD-NEXT: ld1 { v6.b }[14], [x14] -; CHECK-SD-NEXT: add x14, sp, #1392 -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x12] -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] -; CHECK-SD-NEXT: add x11, sp, #1008 -; CHECK-SD-NEXT: ld1 { v0.b }[12], [x8] -; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11] -; CHECK-SD-NEXT: add x8, sp, #1400 -; CHECK-SD-NEXT: ld1 { v3.b }[15], [x10] -; CHECK-SD-NEXT: add x10, sp, #496 -; CHECK-SD-NEXT: add x9, sp, #24 -; CHECK-SD-NEXT: ld1 { v6.b }[15], [x15] -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x8] -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10] -; CHECK-SD-NEXT: add x10, sp, #1016 -; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10] -; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13] -; CHECK-SD-NEXT: add x8, sp, #1408 -; CHECK-SD-NEXT: ld1 { v1.b }[9], [x9] -; CHECK-SD-NEXT: add x9, sp, #504 -; CHECK-SD-NEXT: add x10, sp, #512 -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x8] -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #1024 -; CHECK-SD-NEXT: add x8, sp, #32 -; CHECK-SD-NEXT: ld1 { v16.b }[6], [x9] -; CHECK-SD-NEXT: ld1 { v0.b }[14], [x16] -; CHECK-SD-NEXT: ld1 { v1.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #1416 -; CHECK-SD-NEXT: add x9, sp, #456 -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8] -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x10] -; CHECK-SD-NEXT: add x10, sp, #1032 -; CHECK-SD-NEXT: add x8, sp, #40 -; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10] -; CHECK-SD-NEXT: ld1 { v0.b }[15], [x9] -; CHECK-SD-NEXT: ld1 { v1.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #1424 -; CHECK-SD-NEXT: add x9, sp, #520 -; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8] -; CHECK-SD-NEXT: ld1 { v2.b }[7], [x9] -; CHECK-SD-NEXT: add x9, sp, #1040 -; CHECK-SD-NEXT: add x8, sp, #48 -; CHECK-SD-NEXT: ld1 { v16.b }[8], [x9] -; CHECK-SD-NEXT: add x10, sp, #528 -; CHECK-SD-NEXT: ld1 { v1.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #1432 -; CHECK-SD-NEXT: sdot v5.4s, v6.16b, v3.16b -; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8] -; CHECK-SD-NEXT: ld1 { v2.b }[8], [x10] -; CHECK-SD-NEXT: add x8, sp, #1048 -; CHECK-SD-NEXT: ldr b3, [sp, #80] -; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8] -; CHECK-SD-NEXT: add x10, sp, #88 -; CHECK-SD-NEXT: add x8, sp, #536 -; CHECK-SD-NEXT: add x11, sp, #1440 -; CHECK-SD-NEXT: add x9, sp, #56 -; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10] -; CHECK-SD-NEXT: ld1 { v2.b }[9], [x8] -; CHECK-SD-NEXT: add x8, sp, #1056 -; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11] -; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8] -; CHECK-SD-NEXT: ld1 { v1.b }[13], [x9] -; CHECK-SD-NEXT: add x9, sp, #96 -; CHECK-SD-NEXT: add x8, sp, #544 -; CHECK-SD-NEXT: add x10, sp, #1448 -; CHECK-SD-NEXT: ld1 { v3.b }[2], [x9] -; CHECK-SD-NEXT: ld1 { v2.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #1064 -; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8] -; CHECK-SD-NEXT: add x10, sp, #104 -; CHECK-SD-NEXT: add x8, sp, #552 -; CHECK-SD-NEXT: add x11, sp, #1456 -; CHECK-SD-NEXT: add x9, sp, #64 -; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10] -; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #1072 -; CHECK-SD-NEXT: ld1 { v7.b }[12], [x11] -; CHECK-SD-NEXT: ld1 { v16.b }[12], [x8] -; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9] -; CHECK-SD-NEXT: add x9, sp, #112 -; CHECK-SD-NEXT: add x8, sp, #560 -; CHECK-SD-NEXT: add x10, sp, #1464 -; CHECK-SD-NEXT: ld1 { v3.b }[4], [x9] -; CHECK-SD-NEXT: ld1 { v2.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #1080 -; CHECK-SD-NEXT: ld1 { v7.b }[13], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[13], [x8] -; CHECK-SD-NEXT: add x10, sp, #120 -; CHECK-SD-NEXT: add x8, sp, #568 -; CHECK-SD-NEXT: add x11, sp, #1472 -; CHECK-SD-NEXT: add x9, sp, #72 -; CHECK-SD-NEXT: ld1 { v3.b }[5], [x10] -; CHECK-SD-NEXT: ld1 { v2.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #1088 -; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11] -; CHECK-SD-NEXT: ld1 { v16.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v1.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #128 -; CHECK-SD-NEXT: ldr b6, [sp, #1104] -; CHECK-SD-NEXT: add x10, sp, #1480 -; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9] -; CHECK-SD-NEXT: add x8, sp, #1096 -; CHECK-SD-NEXT: add x9, sp, #1112 -; CHECK-SD-NEXT: ld1 { v7.b }[15], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[15], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9] -; CHECK-SD-NEXT: add x8, sp, #728 -; CHECK-SD-NEXT: add x9, sp, #576 -; CHECK-SD-NEXT: add x10, sp, #136 -; CHECK-SD-NEXT: ld1 { v17.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #1120 -; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9] -; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #736 -; CHECK-SD-NEXT: ldr b7, [sp, #1232] -; CHECK-SD-NEXT: ldr b16, [sp, #848] -; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10] -; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8] -; CHECK-SD-NEXT: add x9, sp, #1240 -; CHECK-SD-NEXT: add x10, sp, #856 -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10] -; CHECK-SD-NEXT: add x8, sp, #1128 -; CHECK-SD-NEXT: add x11, sp, #744 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8] -; CHECK-SD-NEXT: add x10, sp, #1248 -; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11] -; CHECK-SD-NEXT: add x11, sp, #864 -; CHECK-SD-NEXT: add x9, sp, #144 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11] -; CHECK-SD-NEXT: add x8, sp, #1136 -; CHECK-SD-NEXT: add x12, sp, #752 -; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12] -; CHECK-SD-NEXT: add x9, sp, #1256 -; CHECK-SD-NEXT: add x10, sp, #872 -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10] -; CHECK-SD-NEXT: add x8, sp, #1144 -; CHECK-SD-NEXT: add x11, sp, #760 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8] -; CHECK-SD-NEXT: add x10, sp, #1264 -; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11] -; CHECK-SD-NEXT: add x11, sp, #880 -; CHECK-SD-NEXT: add x9, sp, #152 -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11] -; CHECK-SD-NEXT: add x8, sp, #1152 -; CHECK-SD-NEXT: add x12, sp, #768 -; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12] -; CHECK-SD-NEXT: add x9, sp, #1272 -; CHECK-SD-NEXT: add x10, sp, #888 -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10] -; CHECK-SD-NEXT: add x8, sp, #1160 -; CHECK-SD-NEXT: add x11, sp, #776 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8] -; CHECK-SD-NEXT: add x10, sp, #1280 -; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11] -; CHECK-SD-NEXT: add x11, sp, #896 -; CHECK-SD-NEXT: add x9, sp, #160 -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11] -; CHECK-SD-NEXT: add x8, sp, #1168 -; CHECK-SD-NEXT: add x12, sp, #784 -; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12] -; CHECK-SD-NEXT: add x9, sp, #1288 -; CHECK-SD-NEXT: add x10, sp, #904 -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10] -; CHECK-SD-NEXT: add x8, sp, #1176 -; CHECK-SD-NEXT: add x11, sp, #792 -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8] -; CHECK-SD-NEXT: add x10, sp, #1296 -; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11] -; CHECK-SD-NEXT: add x11, sp, #912 -; CHECK-SD-NEXT: add x9, sp, #168 -; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11] -; CHECK-SD-NEXT: add x8, sp, #1184 -; CHECK-SD-NEXT: add x12, sp, #800 -; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12] -; CHECK-SD-NEXT: add x9, sp, #1304 -; CHECK-SD-NEXT: add x10, sp, #920 -; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10] -; CHECK-SD-NEXT: add x8, sp, #1192 -; CHECK-SD-NEXT: add x11, sp, #808 -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8] -; CHECK-SD-NEXT: add x10, sp, #1312 -; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11] -; CHECK-SD-NEXT: add x11, sp, #928 -; CHECK-SD-NEXT: add x9, sp, #176 -; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11] -; CHECK-SD-NEXT: add x8, sp, #1200 -; CHECK-SD-NEXT: add x12, sp, #816 -; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12] -; CHECK-SD-NEXT: add x9, sp, #1320 -; CHECK-SD-NEXT: add x10, sp, #936 -; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10] -; CHECK-SD-NEXT: add x8, sp, #1208 -; CHECK-SD-NEXT: add x11, sp, #824 -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8] -; CHECK-SD-NEXT: add x10, sp, #1328 -; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11] -; CHECK-SD-NEXT: add x11, sp, #944 -; CHECK-SD-NEXT: add x9, sp, #184 -; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11] -; CHECK-SD-NEXT: add x8, sp, #1216 -; CHECK-SD-NEXT: add x12, sp, #832 -; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12] -; CHECK-SD-NEXT: add x9, sp, #1336 -; CHECK-SD-NEXT: add x10, sp, #952 -; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10] -; CHECK-SD-NEXT: add x8, sp, #1224 -; CHECK-SD-NEXT: add x11, sp, #840 -; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #192 -; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11] -; CHECK-SD-NEXT: add x10, sp, #1344 -; CHECK-SD-NEXT: add x11, sp, #960 -; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11] -; CHECK-SD-NEXT: add x9, sp, #584 -; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b -; CHECK-SD-NEXT: add x8, sp, #200 -; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b -; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #1352 -; CHECK-SD-NEXT: add x10, sp, #968 -; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9] -; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10] -; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b -; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b -; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s -; CHECK-SD-NEXT: addv s0, v0.4s -; CHECK-SD-NEXT: fmov w0, s0 -; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_sdot_v48i8_double: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 -; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: ldr w11, [sp, #80] -; CHECK-GI-NEXT: ldr w10, [sp, #208] -; CHECK-GI-NEXT: fmov s0, w0 -; CHECK-GI-NEXT: ldr w8, [sp, #88] -; CHECK-GI-NEXT: ldr w12, [sp, #344] -; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 -; CHECK-GI-NEXT: fmov s1, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #336] -; CHECK-GI-NEXT: fmov s2, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #464] -; CHECK-GI-NEXT: ldr w9, [sp, #216] -; CHECK-GI-NEXT: mov v0.b[1], w1 -; CHECK-GI-NEXT: fmov s3, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #600] -; CHECK-GI-NEXT: movi v21.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v1.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #592] -; CHECK-GI-NEXT: fmov s4, w10 -; CHECK-GI-NEXT: mov v2.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #472] -; CHECK-GI-NEXT: ldr w10, [sp, #608] -; CHECK-GI-NEXT: mov v3.b[1], w12 -; CHECK-GI-NEXT: fmov s5, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #96] -; CHECK-GI-NEXT: mov v4.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #224] -; CHECK-GI-NEXT: mov v0.b[2], w2 -; CHECK-GI-NEXT: mov v1.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #352] -; CHECK-GI-NEXT: ldr w12, [sp, #848] -; CHECK-GI-NEXT: mov v2.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #480] -; CHECK-GI-NEXT: mov v5.b[1], w11 -; CHECK-GI-NEXT: mov v3.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #104] -; CHECK-GI-NEXT: ldr w11, [sp, #16] -; CHECK-GI-NEXT: mov v4.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #232] -; CHECK-GI-NEXT: mov v0.b[3], w3 -; CHECK-GI-NEXT: mov v1.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #360] -; CHECK-GI-NEXT: fmov s7, w12 -; CHECK-GI-NEXT: mov v2.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #488] -; CHECK-GI-NEXT: mov v5.b[2], w10 -; CHECK-GI-NEXT: mov v3.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #112] -; CHECK-GI-NEXT: ldr w10, [sp, #616] -; CHECK-GI-NEXT: mov v4.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #240] -; CHECK-GI-NEXT: mov v0.b[4], w4 -; CHECK-GI-NEXT: mov v1.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #368] -; CHECK-GI-NEXT: ldr w12, [sp, #1112] -; CHECK-GI-NEXT: mov v2.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #496] -; CHECK-GI-NEXT: mov v5.b[3], w10 -; CHECK-GI-NEXT: mov v3.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #120] -; CHECK-GI-NEXT: ldr w10, [sp, #624] -; CHECK-GI-NEXT: mov v4.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #248] -; CHECK-GI-NEXT: mov v0.b[5], w5 -; CHECK-GI-NEXT: mov v1.b[5], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #376] -; CHECK-GI-NEXT: movi v22.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #504] -; CHECK-GI-NEXT: mov v5.b[4], w10 -; CHECK-GI-NEXT: mov v3.b[5], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #128] -; CHECK-GI-NEXT: ldr w10, [sp, #632] -; CHECK-GI-NEXT: mov v4.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #256] -; CHECK-GI-NEXT: mov v0.b[6], w6 -; CHECK-GI-NEXT: mov v1.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #384] -; CHECK-GI-NEXT: movi v23.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #512] -; CHECK-GI-NEXT: mov v5.b[5], w10 -; CHECK-GI-NEXT: mov v3.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #136] -; CHECK-GI-NEXT: ldr w10, [sp, #640] -; CHECK-GI-NEXT: mov v4.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #264] -; CHECK-GI-NEXT: mov v0.b[7], w7 -; CHECK-GI-NEXT: mov v1.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #392] -; CHECK-GI-NEXT: movi v24.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #520] -; CHECK-GI-NEXT: mov v5.b[6], w10 -; CHECK-GI-NEXT: mov v3.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #144] -; CHECK-GI-NEXT: ldr w10, [sp, #648] -; CHECK-GI-NEXT: mov v4.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #272] -; CHECK-GI-NEXT: mov v0.b[8], w11 -; CHECK-GI-NEXT: mov v1.b[8], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #400] -; CHECK-GI-NEXT: ldr w11, [sp, #24] -; CHECK-GI-NEXT: mov v2.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #528] -; CHECK-GI-NEXT: mov v5.b[7], w10 -; CHECK-GI-NEXT: mov v3.b[8], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #152] -; CHECK-GI-NEXT: ldr w10, [sp, #656] -; CHECK-GI-NEXT: mov v4.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #280] -; CHECK-GI-NEXT: mov v0.b[9], w11 -; CHECK-GI-NEXT: mov v1.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #408] -; CHECK-GI-NEXT: ldr w11, [sp, #32] -; CHECK-GI-NEXT: mov v2.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #536] -; CHECK-GI-NEXT: mov v5.b[8], w10 -; CHECK-GI-NEXT: mov v3.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #160] -; CHECK-GI-NEXT: ldr w10, [sp, #664] -; CHECK-GI-NEXT: mov v4.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #288] -; CHECK-GI-NEXT: mov v0.b[10], w11 -; CHECK-GI-NEXT: mov v1.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #416] -; CHECK-GI-NEXT: ldr w11, [sp, #40] -; CHECK-GI-NEXT: mov v2.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #544] -; CHECK-GI-NEXT: mov v5.b[9], w10 -; CHECK-GI-NEXT: mov v3.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #168] -; CHECK-GI-NEXT: ldr w10, [sp, #672] -; CHECK-GI-NEXT: mov v4.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #296] -; CHECK-GI-NEXT: mov v0.b[11], w11 -; CHECK-GI-NEXT: mov v1.b[11], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #424] -; CHECK-GI-NEXT: ldr w11, [sp, #48] -; CHECK-GI-NEXT: mov v2.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #552] -; CHECK-GI-NEXT: mov v5.b[10], w10 -; CHECK-GI-NEXT: mov v3.b[11], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #176] -; CHECK-GI-NEXT: ldr w10, [sp, #680] -; CHECK-GI-NEXT: mov v4.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #304] -; CHECK-GI-NEXT: mov v0.b[12], w11 -; CHECK-GI-NEXT: mov v1.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #432] -; CHECK-GI-NEXT: ldr w11, [sp, #56] -; CHECK-GI-NEXT: mov v2.b[12], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #560] -; CHECK-GI-NEXT: mov v5.b[11], w10 -; CHECK-GI-NEXT: mov v3.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #184] -; CHECK-GI-NEXT: ldr w10, [sp, #688] -; CHECK-GI-NEXT: mov v4.b[12], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #312] -; CHECK-GI-NEXT: mov v0.b[13], w11 -; CHECK-GI-NEXT: mov v1.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #440] -; CHECK-GI-NEXT: ldr w11, [sp, #64] -; CHECK-GI-NEXT: mov v2.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #568] -; CHECK-GI-NEXT: mov v5.b[12], w10 -; CHECK-GI-NEXT: mov v3.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #192] -; CHECK-GI-NEXT: ldr w10, [sp, #696] -; CHECK-GI-NEXT: mov v4.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #320] -; CHECK-GI-NEXT: mov v0.b[14], w11 -; CHECK-GI-NEXT: mov v1.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #448] -; CHECK-GI-NEXT: ldr w11, [sp, #72] -; CHECK-GI-NEXT: mov v2.b[14], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #576] -; CHECK-GI-NEXT: mov v5.b[13], w10 -; CHECK-GI-NEXT: mov v3.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #720] -; CHECK-GI-NEXT: ldr w10, [sp, #704] -; CHECK-GI-NEXT: mov v4.b[14], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #728] -; CHECK-GI-NEXT: mov v0.b[15], w11 -; CHECK-GI-NEXT: fmov s6, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #328] -; CHECK-GI-NEXT: ldr w11, [sp, #456] -; CHECK-GI-NEXT: mov v5.b[14], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #200] -; CHECK-GI-NEXT: movi v25.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v2.b[15], w8 -; CHECK-GI-NEXT: mov v3.b[15], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #736] -; CHECK-GI-NEXT: mov v6.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #584] -; CHECK-GI-NEXT: ldr w8, [sp, #856] -; CHECK-GI-NEXT: mov v1.b[15], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #712] -; CHECK-GI-NEXT: mov v4.b[15], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #976] -; CHECK-GI-NEXT: mov v7.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1232] -; CHECK-GI-NEXT: mov v5.b[15], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #984] -; CHECK-GI-NEXT: mov v6.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1104] -; CHECK-GI-NEXT: fmov s16, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1360] -; CHECK-GI-NEXT: fmov s18, w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1368] -; CHECK-GI-NEXT: fmov s17, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1240] -; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b -; CHECK-GI-NEXT: mov v16.b[1], w10 -; CHECK-GI-NEXT: fmov s19, w9 -; CHECK-GI-NEXT: ldr w10, [sp, #864] -; CHECK-GI-NEXT: mov v18.b[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #992] -; CHECK-GI-NEXT: ldr w9, [sp, #1120] -; CHECK-GI-NEXT: mov v17.b[1], w12 -; CHECK-GI-NEXT: mov v7.b[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1248] -; CHECK-GI-NEXT: mov v19.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #744] -; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b -; CHECK-GI-NEXT: mov v16.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #872] -; CHECK-GI-NEXT: addv s0, v20.4s -; CHECK-GI-NEXT: mov v6.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1000] -; CHECK-GI-NEXT: mov v18.b[2], w10 -; CHECK-GI-NEXT: mov v17.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1376] -; CHECK-GI-NEXT: ldr w10, [sp, #1128] -; CHECK-GI-NEXT: mov v7.b[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #880] -; CHECK-GI-NEXT: addv s1, v21.4s -; CHECK-GI-NEXT: mov v19.b[2], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #752] -; CHECK-GI-NEXT: mov v16.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1256] -; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b -; CHECK-GI-NEXT: mov v17.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1384] -; CHECK-GI-NEXT: mov v6.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1008] -; CHECK-GI-NEXT: mov v18.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1136] -; CHECK-GI-NEXT: mov v19.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #760] -; CHECK-GI-NEXT: mov v7.b[4], w11 -; CHECK-GI-NEXT: mov v16.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1264] -; CHECK-GI-NEXT: ldr w11, [sp, #888] -; CHECK-GI-NEXT: mov v17.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1392] -; CHECK-GI-NEXT: mov v6.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1016] -; CHECK-GI-NEXT: mov v18.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1144] -; CHECK-GI-NEXT: mov v19.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #768] -; CHECK-GI-NEXT: mov v7.b[5], w11 -; CHECK-GI-NEXT: mov v16.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1272] -; CHECK-GI-NEXT: ldr w11, [sp, #896] -; CHECK-GI-NEXT: mov v17.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1400] -; CHECK-GI-NEXT: mov v6.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1024] -; CHECK-GI-NEXT: mov v18.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1152] -; CHECK-GI-NEXT: mov v19.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #776] -; CHECK-GI-NEXT: mov v7.b[6], w11 -; CHECK-GI-NEXT: mov v16.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1280] -; CHECK-GI-NEXT: ldr w11, [sp, #904] -; CHECK-GI-NEXT: mov v17.b[6], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1408] -; CHECK-GI-NEXT: mov v6.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1032] -; CHECK-GI-NEXT: mov v18.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1160] -; CHECK-GI-NEXT: mov v19.b[6], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #784] -; CHECK-GI-NEXT: mov v7.b[7], w11 -; CHECK-GI-NEXT: mov v16.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1288] -; CHECK-GI-NEXT: ldr w11, [sp, #912] -; CHECK-GI-NEXT: mov v17.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1416] -; CHECK-GI-NEXT: mov v6.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1040] -; CHECK-GI-NEXT: mov v18.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1168] -; CHECK-GI-NEXT: mov v19.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #792] -; CHECK-GI-NEXT: mov v7.b[8], w11 -; CHECK-GI-NEXT: mov v16.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1296] -; CHECK-GI-NEXT: ldr w11, [sp, #920] -; CHECK-GI-NEXT: mov v17.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1424] -; CHECK-GI-NEXT: mov v6.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1048] -; CHECK-GI-NEXT: mov v18.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1176] -; CHECK-GI-NEXT: mov v19.b[8], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #800] -; CHECK-GI-NEXT: mov v7.b[9], w11 -; CHECK-GI-NEXT: mov v16.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1304] -; CHECK-GI-NEXT: ldr w11, [sp, #928] -; CHECK-GI-NEXT: mov v17.b[9], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1432] -; CHECK-GI-NEXT: mov v6.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1056] -; CHECK-GI-NEXT: mov v18.b[9], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1184] -; CHECK-GI-NEXT: mov v19.b[9], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #808] -; CHECK-GI-NEXT: mov v7.b[10], w11 -; CHECK-GI-NEXT: mov v16.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1312] -; CHECK-GI-NEXT: ldr w11, [sp, #936] -; CHECK-GI-NEXT: mov v17.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1440] -; CHECK-GI-NEXT: mov v6.b[11], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1064] -; CHECK-GI-NEXT: mov v18.b[10], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1192] -; CHECK-GI-NEXT: mov v19.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #816] -; CHECK-GI-NEXT: mov v7.b[11], w11 -; CHECK-GI-NEXT: mov v16.b[11], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1320] -; CHECK-GI-NEXT: ldr w11, [sp, #944] -; CHECK-GI-NEXT: mov v17.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1448] -; CHECK-GI-NEXT: mov v6.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1072] -; CHECK-GI-NEXT: mov v18.b[11], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1200] -; CHECK-GI-NEXT: mov v19.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #824] -; CHECK-GI-NEXT: mov v7.b[12], w11 -; CHECK-GI-NEXT: mov v16.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1328] -; CHECK-GI-NEXT: ldr w11, [sp, #952] -; CHECK-GI-NEXT: mov v17.b[12], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1456] -; CHECK-GI-NEXT: mov v6.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1080] -; CHECK-GI-NEXT: mov v18.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1208] -; CHECK-GI-NEXT: mov v19.b[12], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #832] -; CHECK-GI-NEXT: mov v7.b[13], w11 -; CHECK-GI-NEXT: mov v16.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1336] -; CHECK-GI-NEXT: ldr w11, [sp, #960] -; CHECK-GI-NEXT: mov v17.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1464] -; CHECK-GI-NEXT: mov v6.b[14], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1088] -; CHECK-GI-NEXT: mov v18.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1216] -; CHECK-GI-NEXT: mov v19.b[13], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #840] -; CHECK-GI-NEXT: mov v7.b[14], w11 -; CHECK-GI-NEXT: mov v16.b[14], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1344] -; CHECK-GI-NEXT: ldr w11, [sp, #968] -; CHECK-GI-NEXT: mov v17.b[14], w9 -; CHECK-GI-NEXT: mov v6.b[15], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1096] -; CHECK-GI-NEXT: mov v18.b[14], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #1472] -; CHECK-GI-NEXT: ldr w10, [sp, #1224] -; CHECK-GI-NEXT: mov v7.b[15], w11 -; CHECK-GI-NEXT: addv s4, v25.4s -; CHECK-GI-NEXT: mov v16.b[15], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #1352] -; CHECK-GI-NEXT: mov v19.b[14], w9 -; CHECK-GI-NEXT: mov v17.b[15], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #1480] -; CHECK-GI-NEXT: mov v18.b[15], w8 +; CHECK-NEXT: add x10, sp, #48 +; CHECK-NEXT: ld1 { v4.b }[1], [x12] +; CHECK-NEXT: add x12, sp, #176 +; CHECK-NEXT: ld1 { v5.b }[2], [x13] +; CHECK-NEXT: add x13, sp, #680 +; CHECK-NEXT: ld1 { v3.b }[4], [x10] +; CHECK-NEXT: ld1 { v2.b }[4], [x12] +; CHECK-NEXT: ld1 { v6.b }[1], [x13] +; CHECK-NEXT: add x13, sp, #56 +; CHECK-NEXT: ld1 { v0.b }[5], [x9] +; CHECK-NEXT: mov v1.b[2], w2 +; CHECK-NEXT: add x8, sp, #128 +; CHECK-NEXT: add x14, sp, #184 +; CHECK-NEXT: add x11, sp, #136 +; CHECK-NEXT: ld1 { v3.b }[5], [x13] +; CHECK-NEXT: add x13, sp, #552 +; CHECK-NEXT: ld1 { v2.b }[5], [x14] +; CHECK-NEXT: ld1 { v16.b }[1], [x13] +; CHECK-NEXT: add x14, sp, #624 +; CHECK-NEXT: ld1 { v0.b }[6], [x8] +; CHECK-NEXT: add x8, sp, #688 +; CHECK-NEXT: add x13, sp, #504 +; CHECK-NEXT: ld1 { v4.b }[2], [x14] +; CHECK-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-NEXT: add x8, sp, #560 +; CHECK-NEXT: ld1 { v5.b }[3], [x13] +; CHECK-NEXT: ld1 { v16.b }[2], [x8] +; CHECK-NEXT: mov v1.b[3], w3 +; CHECK-NEXT: add x9, sp, #64 +; CHECK-NEXT: add x15, sp, #632 +; CHECK-NEXT: ld1 { v3.b }[6], [x9] +; CHECK-NEXT: ld1 { v0.b }[7], [x11] +; CHECK-NEXT: ld1 { v4.b }[3], [x15] +; CHECK-NEXT: add x8, sp, #696 +; CHECK-NEXT: add x9, sp, #568 +; CHECK-NEXT: add x11, sp, #512 +; CHECK-NEXT: ld1 { v6.b }[3], [x8] +; CHECK-NEXT: ld1 { v16.b }[3], [x9] +; CHECK-NEXT: ld1 { v5.b }[4], [x11] +; CHECK-NEXT: add x8, sp, #640 +; CHECK-NEXT: mov v1.b[4], w4 +; CHECK-NEXT: ld1 { v4.b }[4], [x8] +; CHECK-NEXT: add x8, sp, #704 +; CHECK-NEXT: add x9, sp, #576 +; CHECK-NEXT: add x11, sp, #520 +; CHECK-NEXT: ld1 { v6.b }[4], [x8] +; CHECK-NEXT: ld1 { v16.b }[4], [x9] +; CHECK-NEXT: ld1 { v5.b }[5], [x11] +; CHECK-NEXT: ldr b18, [sp, #736] +; CHECK-NEXT: add x12, sp, #192 +; CHECK-NEXT: ld1 { v2.b }[6], [x12] +; CHECK-NEXT: add x8, sp, #648 +; CHECK-NEXT: add x9, sp, #528 +; CHECK-NEXT: add x11, sp, #712 +; CHECK-NEXT: add x12, sp, #584 +; CHECK-NEXT: sshll v18.8h, v18.8b, #0 +; CHECK-NEXT: mov v1.b[5], w5 +; CHECK-NEXT: ld1 { v6.b }[5], [x11] +; CHECK-NEXT: ld1 { v16.b }[5], [x12] +; CHECK-NEXT: ld1 { v4.b }[5], [x8] +; CHECK-NEXT: ld1 { v5.b }[6], [x9] +; CHECK-NEXT: movi v17.2d, #0000000000000000 +; CHECK-NEXT: add x8, sp, #656 +; CHECK-NEXT: add x9, sp, #536 +; CHECK-NEXT: add x11, sp, #720 +; CHECK-NEXT: add x12, sp, #592 +; CHECK-NEXT: sshll v18.4s, v18.4h, #0 +; CHECK-NEXT: ldr b7, [sp, #208] +; CHECK-NEXT: ld1 { v6.b }[6], [x11] +; CHECK-NEXT: ld1 { v16.b }[6], [x12] +; CHECK-NEXT: ld1 { v4.b }[6], [x8] +; CHECK-NEXT: ld1 { v5.b }[7], [x9] +; CHECK-NEXT: mov v1.b[6], w6 +; CHECK-NEXT: sshll v7.8h, v7.8b, #0 +; CHECK-NEXT: add x8, sp, #664 +; CHECK-NEXT: add x9, sp, #728 +; CHECK-NEXT: add x11, sp, #600 +; CHECK-NEXT: mov v17.s[0], v18.s[0] +; CHECK-NEXT: ld1 { v6.b }[7], [x9] +; CHECK-NEXT: ld1 { v16.b }[7], [x11] +; CHECK-NEXT: ld1 { v4.b }[7], [x8] +; CHECK-NEXT: sshll v5.8h, v5.8b, #0 +; CHECK-NEXT: movi v18.2d, #0000000000000000 +; CHECK-NEXT: add x10, sp, #200 +; CHECK-NEXT: mov v1.b[7], w7 +; CHECK-NEXT: add x9, sp, #72 +; CHECK-NEXT: sshll v7.4s, v7.4h, #0 +; CHECK-NEXT: ld1 { v2.b }[7], [x10] +; CHECK-NEXT: ld1 { v3.b }[7], [x9] +; CHECK-NEXT: sshll v6.8h, v6.8b, #0 +; CHECK-NEXT: sshll v16.8h, v16.8b, #0 +; CHECK-NEXT: sshll v4.8h, v4.8b, #0 +; CHECK-NEXT: saddw v17.4s, v17.4s, v5.4h +; CHECK-NEXT: sshll v0.8h, v0.8b, #0 +; CHECK-NEXT: mov v18.s[0], v7.s[0] +; CHECK-NEXT: sshll v1.8h, v1.8b, #0 +; CHECK-NEXT: sshll v2.8h, v2.8b, #0 +; CHECK-NEXT: sshll v3.8h, v3.8b, #0 +; CHECK-NEXT: saddl2 v7.4s, v16.8h, v6.8h +; CHECK-NEXT: saddl2 v5.4s, v5.8h, v4.8h +; CHECK-NEXT: saddl v6.4s, v16.4h, v6.4h +; CHECK-NEXT: saddw v4.4s, v17.4s, v4.4h +; CHECK-NEXT: saddl2 v17.4s, v1.8h, v0.8h +; CHECK-NEXT: saddl2 v16.4s, v3.8h, v2.8h +; CHECK-NEXT: saddw v1.4s, v18.4s, v1.4h +; CHECK-NEXT: add v5.4s, v5.4s, v7.4s +; CHECK-NEXT: add v4.4s, v4.4s, v6.4s +; CHECK-NEXT: saddl v2.4s, v3.4h, v2.4h +; CHECK-NEXT: add v6.4s, v17.4s, v16.4s +; CHECK-NEXT: saddw v0.4s, v1.4s, v0.4h +; CHECK-NEXT: add v1.4s, v4.4s, v5.4s +; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: add v1.4s, v6.4s, v1.4s +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s +; CHECK-NEXT: addv s0, v0.4s +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret +entry: + %az = sext <33 x i8> %a to <33 x i32> + %r1 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %az) + %cz = sext <33 x i8> %c to <33 x i32> + %r2 = call i32 @llvm.vector.reduce.add.v33i32(<33 x i32> %cz) + %x = add i32 %r1, %r2 + ret i32 %x +} + +define i32 @test_udot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_udot_v48i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q1, [x0, #32] +; CHECK-SD-NEXT: ldr q2, [x1, #32] +; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b +; CHECK-SD-NEXT: ldp q3, q1, [x0] +; CHECK-SD-NEXT: ldp q4, q2, [x1] +; CHECK-SD-NEXT: udot v0.4s, v4.16b, v3.16b +; CHECK-SD-NEXT: udot v0.4s, v2.16b, v1.16b +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_udot_v48i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v0.2d, #0000000000000000 +; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 +; CHECK-GI-NEXT: ldr q7, [x0, #32] +; CHECK-GI-NEXT: ldp q3, q4, [x0] +; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-NEXT: ldp q5, q6, [x1] +; CHECK-GI-NEXT: ldr q16, [x1, #32] +; CHECK-GI-NEXT: udot v0.4s, v5.16b, v3.16b +; CHECK-GI-NEXT: udot v1.4s, v6.16b, v4.16b +; CHECK-GI-NEXT: udot v2.4s, v16.16b, v7.16b +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: addv s2, v2.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w0, w8, w2 +; CHECK-GI-NEXT: ret +entry: + %0 = load <48 x i8>, ptr %a + %1 = zext <48 x i8> %0 to <48 x i32> + %2 = load <48 x i8>, ptr %b + %3 = zext <48 x i8> %2 to <48 x i32> + %4 = mul nuw nsw <48 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4) + %op.extra = add i32 %5, %sum + ret i32 %op.extra +} + +define i32 @test_udot_v48i8_nomla(ptr nocapture readonly %a1) { +; CHECK-SD-LABEL: test_udot_v48i8_nomla: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.16b, #1 +; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q2, [x0, #32] +; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b +; CHECK-SD-NEXT: ldp q3, q2, [x0] +; CHECK-SD-NEXT: udot v1.4s, v3.16b, v0.16b +; CHECK-SD-NEXT: udot v1.4s, v2.16b, v0.16b +; CHECK-SD-NEXT: addv s0, v1.4s +; CHECK-SD-NEXT: fmov w0, s0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_udot_v48i8_nomla: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v0.16b, #1 +; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 +; CHECK-GI-NEXT: ldr q6, [x0, #32] +; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-NEXT: ldp q4, q5, [x0] +; CHECK-GI-NEXT: movi v3.2d, #0000000000000000 +; CHECK-GI-NEXT: udot v1.4s, v4.16b, v0.16b +; CHECK-GI-NEXT: udot v2.4s, v5.16b, v0.16b +; CHECK-GI-NEXT: udot v3.4s, v6.16b, v0.16b +; CHECK-GI-NEXT: addv s0, v1.4s +; CHECK-GI-NEXT: addv s1, v2.4s +; CHECK-GI-NEXT: addv s2, v3.4s +; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w9, s1 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: add w0, w8, w9 +; CHECK-GI-NEXT: ret +entry: + %0 = load <48 x i8>, ptr %a1 + %1 = zext <48 x i8> %0 to <48 x i32> + %2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %1) + ret i32 %2 +} +define i32 @test_sdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { +; CHECK-SD-LABEL: test_sdot_v48i8: +; CHECK-SD: // %bb.0: // %entry +; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 +; CHECK-SD-NEXT: ldr q1, [x0, #32] +; CHECK-SD-NEXT: ldr q2, [x1, #32] +; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b +; CHECK-SD-NEXT: ldp q3, q1, [x0] +; CHECK-SD-NEXT: ldp q4, q2, [x1] +; CHECK-SD-NEXT: sdot v0.4s, v4.16b, v3.16b +; CHECK-SD-NEXT: sdot v0.4s, v2.16b, v1.16b +; CHECK-SD-NEXT: addv s0, v0.4s +; CHECK-SD-NEXT: fmov w8, s0 +; CHECK-SD-NEXT: add w0, w8, w2 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: test_sdot_v48i8: +; CHECK-GI: // %bb.0: // %entry +; CHECK-GI-NEXT: movi v0.2d, #0000000000000000 +; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 +; CHECK-GI-NEXT: ldr q7, [x0, #32] +; CHECK-GI-NEXT: ldp q3, q4, [x0] +; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 +; CHECK-GI-NEXT: ldp q5, q6, [x1] +; CHECK-GI-NEXT: ldr q16, [x1, #32] +; CHECK-GI-NEXT: sdot v0.4s, v5.16b, v3.16b +; CHECK-GI-NEXT: sdot v1.4s, v6.16b, v4.16b +; CHECK-GI-NEXT: sdot v2.4s, v16.16b, v7.16b +; CHECK-GI-NEXT: addv s0, v0.4s +; CHECK-GI-NEXT: addv s1, v1.4s +; CHECK-GI-NEXT: addv s2, v2.4s ; CHECK-GI-NEXT: fmov w8, s0 -; CHECK-GI-NEXT: fmov w11, s4 -; CHECK-GI-NEXT: mov v19.b[15], w9 ; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b -; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b ; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b -; CHECK-GI-NEXT: add w8, w8, w11 -; CHECK-GI-NEXT: addv s2, v22.4s -; CHECK-GI-NEXT: addv s3, v23.4s -; CHECK-GI-NEXT: addv s5, v24.4s ; CHECK-GI-NEXT: fmov w9, s2 -; CHECK-GI-NEXT: fmov w10, s3 -; CHECK-GI-NEXT: add w9, w9, w10 -; CHECK-GI-NEXT: fmov w10, s5 -; CHECK-GI-NEXT: add w9, w9, w10 -; CHECK-GI-NEXT: add w0, w8, w9 -; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: add w0, w8, w2 ; CHECK-GI-NEXT: ret entry: - %az = sext <48 x i8> %a to <48 x i32> - %bz = sext <48 x i8> %b to <48 x i32> - %m1 = mul nuw nsw <48 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1) - %cz = sext <48 x i8> %c to <48 x i32> - %dz = sext <48 x i8> %d to <48 x i32> - %m2 = mul nuw nsw <48 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2) - %x = add i32 %r1, %r2 - ret i32 %x + %0 = load <48 x i8>, ptr %a + %1 = sext <48 x i8> %0 to <48 x i32> + %2 = load <48 x i8>, ptr %b + %3 = sext <48 x i8> %2 to <48 x i32> + %4 = mul nsw <48 x i32> %3, %1 + %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4) + %op.extra = add nsw i32 %5, %sum + ret i32 %op.extra } -define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { -; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla: +define i32 @test_sdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { +; CHECK-SD-LABEL: test_sdot_v48i8_double: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 ; CHECK-SD-NEXT: .cfi_offset w29, -16 -; CHECK-SD-NEXT: ldr b5, [sp, #208] +; CHECK-SD-NEXT: ldr b3, [sp, #592] +; CHECK-SD-NEXT: add x8, sp, #600 +; CHECK-SD-NEXT: ldr b6, [sp, #208] +; CHECK-SD-NEXT: ldr b0, [sp, #336] +; CHECK-SD-NEXT: add x9, sp, #344 +; CHECK-SD-NEXT: ldr b2, [sp, #464] +; CHECK-SD-NEXT: ld1 { v3.b }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #216 -; CHECK-SD-NEXT: fmov s0, w0 -; CHECK-SD-NEXT: ldr b4, [sp, #976] -; CHECK-SD-NEXT: add x9, sp, #984 -; CHECK-SD-NEXT: add x12, sp, #328 -; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8] +; CHECK-SD-NEXT: add x10, sp, #624 +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #608 +; CHECK-SD-NEXT: ld1 { v0.b }[1], [x9] +; CHECK-SD-NEXT: add x9, sp, #232 +; CHECK-SD-NEXT: fmov s1, w0 +; CHECK-SD-NEXT: ldr b7, [sp, #1360] +; CHECK-SD-NEXT: ld1 { v3.b }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #224 -; CHECK-SD-NEXT: movi v1.16b, #1 -; CHECK-SD-NEXT: mov v0.b[1], w1 -; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9] -; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 -; CHECK-SD-NEXT: add x11, sp, #992 -; CHECK-SD-NEXT: ldr b6, [sp, #720] -; CHECK-SD-NEXT: ldr b7, [sp, #80] -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #232 -; CHECK-SD-NEXT: add x13, sp, #88 -; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11] -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13] -; CHECK-SD-NEXT: add x13, sp, #856 -; CHECK-SD-NEXT: mov v0.b[2], w2 -; CHECK-SD-NEXT: add x14, sp, #1008 -; CHECK-SD-NEXT: add x15, sp, #872 -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #240 -; CHECK-SD-NEXT: add x16, sp, #888 -; CHECK-SD-NEXT: add x10, sp, #16 -; CHECK-SD-NEXT: add x9, sp, #24 -; CHECK-SD-NEXT: add x11, sp, #40 -; CHECK-SD-NEXT: movi v2.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #248 -; CHECK-SD-NEXT: mov v0.b[3], w3 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #256 -; CHECK-SD-NEXT: mov v0.b[4], w4 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #264 -; CHECK-SD-NEXT: mov v0.b[5], w5 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8] -; CHECK-SD-NEXT: add x8, sp, #272 -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8] -; CHECK-SD-NEXT: add x8, sp, #280 -; CHECK-SD-NEXT: mov v0.b[6], w6 -; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8] -; CHECK-SD-NEXT: add x8, sp, #288 -; CHECK-SD-NEXT: mov v0.b[7], w7 -; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #296 -; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10] -; CHECK-SD-NEXT: add x10, sp, #128 -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #304 -; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9] -; CHECK-SD-NEXT: add x9, sp, #136 -; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #312 -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #320 -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] -; CHECK-SD-NEXT: add x8, sp, #32 -; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #144 -; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12] -; CHECK-SD-NEXT: add x12, sp, #728 -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] -; CHECK-SD-NEXT: add x12, sp, #1000 -; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11] -; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12] -; CHECK-SD-NEXT: add x12, sp, #736 -; CHECK-SD-NEXT: add x11, sp, #920 -; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b -; CHECK-SD-NEXT: ldr b5, [sp, #848] -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12] -; CHECK-SD-NEXT: add x12, sp, #48 -; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13] -; CHECK-SD-NEXT: add x13, sp, #744 -; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14] -; CHECK-SD-NEXT: add x14, sp, #96 -; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13] -; CHECK-SD-NEXT: add x13, sp, #864 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14] -; CHECK-SD-NEXT: add x14, sp, #1016 -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13] -; CHECK-SD-NEXT: add x13, sp, #752 -; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14] -; CHECK-SD-NEXT: add x14, sp, #104 -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13] -; CHECK-SD-NEXT: add x13, sp, #1024 -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14] -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15] -; CHECK-SD-NEXT: add x15, sp, #760 -; CHECK-SD-NEXT: add x14, sp, #112 -; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] -; CHECK-SD-NEXT: add x13, sp, #880 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15] -; CHECK-SD-NEXT: add x15, sp, #1032 -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13] -; CHECK-SD-NEXT: add x14, sp, #768 -; CHECK-SD-NEXT: add x13, sp, #120 -; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15] -; CHECK-SD-NEXT: add x15, sp, #1040 -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14] -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13] -; CHECK-SD-NEXT: add x13, sp, #776 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16] -; CHECK-SD-NEXT: add x14, sp, #1048 -; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15] -; CHECK-SD-NEXT: add x15, sp, #896 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13] -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] -; CHECK-SD-NEXT: add x10, sp, #784 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15] -; CHECK-SD-NEXT: add x13, sp, #1056 -; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] -; CHECK-SD-NEXT: add x14, sp, #904 -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10] -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] -; CHECK-SD-NEXT: add x9, sp, #792 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14] -; CHECK-SD-NEXT: add x10, sp, #1064 -; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13] -; CHECK-SD-NEXT: add x13, sp, #912 -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] +; CHECK-SD-NEXT: add x11, sp, #648 +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #616 +; CHECK-SD-NEXT: add x12, sp, #376 +; CHECK-SD-NEXT: mov v1.b[1], w1 +; CHECK-SD-NEXT: ldr b16, [sp, #976] +; CHECK-SD-NEXT: add x14, sp, #288 +; CHECK-SD-NEXT: ld1 { v3.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #632 +; CHECK-SD-NEXT: add x15, sp, #408 +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #472 +; CHECK-SD-NEXT: add x13, sp, #696 +; CHECK-SD-NEXT: ld1 { v2.b }[1], [x9] +; CHECK-SD-NEXT: add x9, sp, #240 +; CHECK-SD-NEXT: add x16, sp, #448 +; CHECK-SD-NEXT: ld1 { v3.b }[4], [x10] +; CHECK-SD-NEXT: add x10, sp, #352 +; CHECK-SD-NEXT: mov v1.b[2], w2 +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x9] +; CHECK-SD-NEXT: ld1 { v0.b }[2], [x10] +; CHECK-SD-NEXT: add x10, sp, #1368 +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x10] +; CHECK-SD-NEXT: add x10, sp, #248 +; CHECK-SD-NEXT: add x9, sp, #640 +; CHECK-SD-NEXT: ld1 { v3.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #656 +; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10] +; CHECK-SD-NEXT: add x10, sp, #360 +; CHECK-SD-NEXT: mov v1.b[3], w3 +; CHECK-SD-NEXT: ld1 { v0.b }[3], [x10] +; CHECK-SD-NEXT: add x10, sp, #256 +; CHECK-SD-NEXT: movi v4.2d, #0000000000000000 +; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9] +; CHECK-SD-NEXT: add x9, sp, #368 +; CHECK-SD-NEXT: ldr b17, [sp, #720] +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x10] +; CHECK-SD-NEXT: add x10, sp, #984 +; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10] +; CHECK-SD-NEXT: add x10, sp, #664 +; CHECK-SD-NEXT: ld1 { v3.b }[7], [x11] +; CHECK-SD-NEXT: add x11, sp, #264 +; CHECK-SD-NEXT: mov v1.b[4], w4 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x11] +; CHECK-SD-NEXT: add x9, sp, #672 +; CHECK-SD-NEXT: add x11, sp, #680 +; CHECK-SD-NEXT: ld1 { v0.b }[5], [x12] +; CHECK-SD-NEXT: add x12, sp, #480 +; CHECK-SD-NEXT: ld1 { v2.b }[2], [x12] +; CHECK-SD-NEXT: add x12, sp, #272 +; CHECK-SD-NEXT: ld1 { v3.b }[8], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x12] +; CHECK-SD-NEXT: add x12, sp, #384 +; CHECK-SD-NEXT: mov v1.b[5], w5 +; CHECK-SD-NEXT: ld1 { v0.b }[6], [x12] +; CHECK-SD-NEXT: add x12, sp, #280 +; CHECK-SD-NEXT: add x8, sp, #688 +; CHECK-SD-NEXT: ld1 { v3.b }[9], [x10] +; CHECK-SD-NEXT: add x10, sp, #1376 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] +; CHECK-SD-NEXT: add x10, sp, #392 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12] +; CHECK-SD-NEXT: ld1 { v0.b }[7], [x10] +; CHECK-SD-NEXT: mov v1.b[6], w6 +; CHECK-SD-NEXT: add x12, sp, #704 +; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9] +; CHECK-SD-NEXT: add x9, sp, #400 +; CHECK-SD-NEXT: add x10, sp, #712 +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x14] +; CHECK-SD-NEXT: add x14, sp, #992 +; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[2], [x14] +; CHECK-SD-NEXT: add x14, sp, #296 +; CHECK-SD-NEXT: ld1 { v3.b }[11], [x11] +; CHECK-SD-NEXT: add x9, sp, #304 +; CHECK-SD-NEXT: add x11, sp, #312 +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x14] +; CHECK-SD-NEXT: mov v1.b[7], w7 +; CHECK-SD-NEXT: add x14, sp, #320 +; CHECK-SD-NEXT: ld1 { v0.b }[9], [x15] +; CHECK-SD-NEXT: add x15, sp, #328 +; CHECK-SD-NEXT: ld1 { v3.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #416 +; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9] +; CHECK-SD-NEXT: add x9, sp, #1384 +; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] +; CHECK-SD-NEXT: add x9, sp, #424 +; CHECK-SD-NEXT: ld1 { v3.b }[13], [x13] +; CHECK-SD-NEXT: add x8, sp, #432 +; CHECK-SD-NEXT: add x13, sp, #440 +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11] +; CHECK-SD-NEXT: add x11, sp, #16 +; CHECK-SD-NEXT: ld1 { v0.b }[11], [x9] +; CHECK-SD-NEXT: add x9, sp, #1000 +; CHECK-SD-NEXT: ld1 { v1.b }[8], [x11] +; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9] +; CHECK-SD-NEXT: ld1 { v3.b }[14], [x12] +; CHECK-SD-NEXT: add x12, sp, #488 +; CHECK-SD-NEXT: ld1 { v6.b }[14], [x14] +; CHECK-SD-NEXT: add x14, sp, #1392 +; CHECK-SD-NEXT: ld1 { v2.b }[3], [x12] +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] +; CHECK-SD-NEXT: add x11, sp, #1008 +; CHECK-SD-NEXT: ld1 { v0.b }[12], [x8] +; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11] +; CHECK-SD-NEXT: add x8, sp, #1400 +; CHECK-SD-NEXT: ld1 { v3.b }[15], [x10] +; CHECK-SD-NEXT: add x10, sp, #496 +; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: ld1 { v6.b }[15], [x15] +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10] +; CHECK-SD-NEXT: add x10, sp, #1016 +; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-SD-NEXT: ld1 { v0.b }[13], [x13] +; CHECK-SD-NEXT: add x8, sp, #1408 +; CHECK-SD-NEXT: ld1 { v1.b }[9], [x9] +; CHECK-SD-NEXT: add x9, sp, #504 +; CHECK-SD-NEXT: add x10, sp, #512 +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] +; CHECK-SD-NEXT: add x9, sp, #1024 +; CHECK-SD-NEXT: add x8, sp, #32 +; CHECK-SD-NEXT: ld1 { v16.b }[6], [x9] +; CHECK-SD-NEXT: ld1 { v0.b }[14], [x16] +; CHECK-SD-NEXT: ld1 { v1.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #1416 +; CHECK-SD-NEXT: add x9, sp, #456 +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x8] +; CHECK-SD-NEXT: ld1 { v2.b }[6], [x10] +; CHECK-SD-NEXT: add x10, sp, #1032 +; CHECK-SD-NEXT: add x8, sp, #40 +; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10] +; CHECK-SD-NEXT: ld1 { v0.b }[15], [x9] +; CHECK-SD-NEXT: ld1 { v1.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #1424 +; CHECK-SD-NEXT: add x9, sp, #520 ; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8] -; CHECK-SD-NEXT: add x9, sp, #800 -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13] -; CHECK-SD-NEXT: add x8, sp, #152 -; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10] -; CHECK-SD-NEXT: add x10, sp, #1072 -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9] +; CHECK-SD-NEXT: ld1 { v2.b }[7], [x9] +; CHECK-SD-NEXT: add x9, sp, #1040 +; CHECK-SD-NEXT: add x8, sp, #48 +; CHECK-SD-NEXT: ld1 { v16.b }[8], [x9] +; CHECK-SD-NEXT: add x10, sp, #528 +; CHECK-SD-NEXT: ld1 { v1.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #1432 +; CHECK-SD-NEXT: sdot v5.4s, v6.16b, v3.16b ; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8] -; CHECK-SD-NEXT: add x9, sp, #808 -; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11] -; CHECK-SD-NEXT: add x8, sp, #56 -; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10] -; CHECK-SD-NEXT: add x10, sp, #160 -; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9] -; CHECK-SD-NEXT: add x9, sp, #928 -; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] -; CHECK-SD-NEXT: add x10, sp, #1080 -; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] -; CHECK-SD-NEXT: add x8, sp, #816 -; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10] +; CHECK-SD-NEXT: ld1 { v2.b }[8], [x10] +; CHECK-SD-NEXT: add x8, sp, #1048 +; CHECK-SD-NEXT: ldr b3, [sp, #80] +; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8] +; CHECK-SD-NEXT: add x10, sp, #88 +; CHECK-SD-NEXT: add x8, sp, #536 +; CHECK-SD-NEXT: add x11, sp, #1440 +; CHECK-SD-NEXT: add x9, sp, #56 +; CHECK-SD-NEXT: ld1 { v3.b }[1], [x10] +; CHECK-SD-NEXT: ld1 { v2.b }[9], [x8] +; CHECK-SD-NEXT: add x8, sp, #1056 +; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11] +; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8] +; CHECK-SD-NEXT: ld1 { v1.b }[13], [x9] +; CHECK-SD-NEXT: add x9, sp, #96 +; CHECK-SD-NEXT: add x8, sp, #544 +; CHECK-SD-NEXT: add x10, sp, #1448 +; CHECK-SD-NEXT: ld1 { v3.b }[2], [x9] +; CHECK-SD-NEXT: ld1 { v2.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #1064 +; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8] +; CHECK-SD-NEXT: add x10, sp, #104 +; CHECK-SD-NEXT: add x8, sp, #552 +; CHECK-SD-NEXT: add x11, sp, #1456 +; CHECK-SD-NEXT: add x9, sp, #64 +; CHECK-SD-NEXT: ld1 { v3.b }[3], [x10] +; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #1072 +; CHECK-SD-NEXT: ld1 { v7.b }[12], [x11] +; CHECK-SD-NEXT: ld1 { v16.b }[12], [x8] +; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9] +; CHECK-SD-NEXT: add x9, sp, #112 +; CHECK-SD-NEXT: add x8, sp, #560 +; CHECK-SD-NEXT: add x10, sp, #1464 +; CHECK-SD-NEXT: ld1 { v3.b }[4], [x9] +; CHECK-SD-NEXT: ld1 { v2.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #1080 +; CHECK-SD-NEXT: ld1 { v7.b }[13], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[13], [x8] +; CHECK-SD-NEXT: add x10, sp, #120 +; CHECK-SD-NEXT: add x8, sp, #568 +; CHECK-SD-NEXT: add x11, sp, #1472 +; CHECK-SD-NEXT: add x9, sp, #72 +; CHECK-SD-NEXT: ld1 { v3.b }[5], [x10] +; CHECK-SD-NEXT: ld1 { v2.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #1088 +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11] +; CHECK-SD-NEXT: ld1 { v16.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v1.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #128 +; CHECK-SD-NEXT: ldr b6, [sp, #1104] +; CHECK-SD-NEXT: add x10, sp, #1480 +; CHECK-SD-NEXT: ld1 { v3.b }[6], [x9] +; CHECK-SD-NEXT: add x8, sp, #1096 +; CHECK-SD-NEXT: add x9, sp, #1112 +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[1], [x9] +; CHECK-SD-NEXT: add x8, sp, #728 +; CHECK-SD-NEXT: add x9, sp, #576 +; CHECK-SD-NEXT: add x10, sp, #136 +; CHECK-SD-NEXT: ld1 { v17.b }[1], [x8] +; CHECK-SD-NEXT: add x8, sp, #1120 +; CHECK-SD-NEXT: ld1 { v2.b }[14], [x9] +; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #736 +; CHECK-SD-NEXT: ldr b7, [sp, #1232] +; CHECK-SD-NEXT: ldr b16, [sp, #848] +; CHECK-SD-NEXT: ld1 { v3.b }[7], [x10] +; CHECK-SD-NEXT: ld1 { v17.b }[2], [x8] +; CHECK-SD-NEXT: add x9, sp, #1240 +; CHECK-SD-NEXT: add x10, sp, #856 +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[1], [x10] +; CHECK-SD-NEXT: add x8, sp, #1128 +; CHECK-SD-NEXT: add x11, sp, #744 +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x8] +; CHECK-SD-NEXT: add x10, sp, #1248 +; CHECK-SD-NEXT: ld1 { v17.b }[3], [x11] +; CHECK-SD-NEXT: add x11, sp, #864 +; CHECK-SD-NEXT: add x9, sp, #144 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[2], [x11] +; CHECK-SD-NEXT: add x8, sp, #1136 +; CHECK-SD-NEXT: add x12, sp, #752 +; CHECK-SD-NEXT: ld1 { v3.b }[8], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[4], [x12] +; CHECK-SD-NEXT: add x9, sp, #1256 +; CHECK-SD-NEXT: add x10, sp, #872 +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[3], [x10] +; CHECK-SD-NEXT: add x8, sp, #1144 +; CHECK-SD-NEXT: add x11, sp, #760 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x8] +; CHECK-SD-NEXT: add x10, sp, #1264 +; CHECK-SD-NEXT: ld1 { v17.b }[5], [x11] +; CHECK-SD-NEXT: add x11, sp, #880 +; CHECK-SD-NEXT: add x9, sp, #152 +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[4], [x11] +; CHECK-SD-NEXT: add x8, sp, #1152 +; CHECK-SD-NEXT: add x12, sp, #768 +; CHECK-SD-NEXT: ld1 { v3.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[6], [x12] +; CHECK-SD-NEXT: add x9, sp, #1272 +; CHECK-SD-NEXT: add x10, sp, #888 +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[5], [x10] +; CHECK-SD-NEXT: add x8, sp, #1160 +; CHECK-SD-NEXT: add x11, sp, #776 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x8] +; CHECK-SD-NEXT: add x10, sp, #1280 +; CHECK-SD-NEXT: ld1 { v17.b }[7], [x11] +; CHECK-SD-NEXT: add x11, sp, #896 +; CHECK-SD-NEXT: add x9, sp, #160 +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[6], [x11] +; CHECK-SD-NEXT: add x8, sp, #1168 +; CHECK-SD-NEXT: add x12, sp, #784 +; CHECK-SD-NEXT: ld1 { v3.b }[10], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[8], [x12] +; CHECK-SD-NEXT: add x9, sp, #1288 +; CHECK-SD-NEXT: add x10, sp, #904 +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[7], [x10] +; CHECK-SD-NEXT: add x8, sp, #1176 +; CHECK-SD-NEXT: add x11, sp, #792 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x8] +; CHECK-SD-NEXT: add x10, sp, #1296 +; CHECK-SD-NEXT: ld1 { v17.b }[9], [x11] +; CHECK-SD-NEXT: add x11, sp, #912 ; CHECK-SD-NEXT: add x9, sp, #168 -; CHECK-SD-NEXT: add x10, sp, #176 +; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[8], [x11] +; CHECK-SD-NEXT: add x8, sp, #1184 +; CHECK-SD-NEXT: add x12, sp, #800 +; CHECK-SD-NEXT: ld1 { v3.b }[11], [x9] +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x8] +; CHECK-SD-NEXT: ld1 { v17.b }[10], [x12] +; CHECK-SD-NEXT: add x9, sp, #1304 +; CHECK-SD-NEXT: add x10, sp, #920 +; CHECK-SD-NEXT: ld1 { v7.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[9], [x10] +; CHECK-SD-NEXT: add x8, sp, #1192 +; CHECK-SD-NEXT: add x11, sp, #808 +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x8] +; CHECK-SD-NEXT: add x10, sp, #1312 +; CHECK-SD-NEXT: ld1 { v17.b }[11], [x11] +; CHECK-SD-NEXT: add x11, sp, #928 +; CHECK-SD-NEXT: add x9, sp, #176 +; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[10], [x11] +; CHECK-SD-NEXT: add x8, sp, #1200 +; CHECK-SD-NEXT: add x12, sp, #816 +; CHECK-SD-NEXT: ld1 { v3.b }[12], [x9] ; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #936 -; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] -; CHECK-SD-NEXT: add x9, sp, #1088 -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #64 -; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] -; CHECK-SD-NEXT: add x9, sp, #824 -; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] -; CHECK-SD-NEXT: add x9, sp, #944 -; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] -; CHECK-SD-NEXT: add x10, sp, #1096 -; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9] -; CHECK-SD-NEXT: add x8, sp, #832 -; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10] +; CHECK-SD-NEXT: ld1 { v17.b }[12], [x12] +; CHECK-SD-NEXT: add x9, sp, #1320 +; CHECK-SD-NEXT: add x10, sp, #936 +; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[11], [x10] +; CHECK-SD-NEXT: add x8, sp, #1208 +; CHECK-SD-NEXT: add x11, sp, #824 +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x8] +; CHECK-SD-NEXT: add x10, sp, #1328 +; CHECK-SD-NEXT: ld1 { v17.b }[13], [x11] +; CHECK-SD-NEXT: add x11, sp, #944 ; CHECK-SD-NEXT: add x9, sp, #184 -; CHECK-SD-NEXT: add x10, sp, #72 +; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[12], [x11] +; CHECK-SD-NEXT: add x8, sp, #1216 +; CHECK-SD-NEXT: add x12, sp, #832 +; CHECK-SD-NEXT: ld1 { v3.b }[13], [x9] ; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] -; CHECK-SD-NEXT: add x8, sp, #952 +; CHECK-SD-NEXT: ld1 { v17.b }[14], [x12] +; CHECK-SD-NEXT: add x9, sp, #1336 +; CHECK-SD-NEXT: add x10, sp, #952 ; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #840 -; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] -; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b -; CHECK-SD-NEXT: add x9, sp, #192 +; CHECK-SD-NEXT: ld1 { v16.b }[13], [x10] +; CHECK-SD-NEXT: add x8, sp, #1224 +; CHECK-SD-NEXT: add x11, sp, #840 ; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #960 -; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9] -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] -; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b +; CHECK-SD-NEXT: add x8, sp, #192 +; CHECK-SD-NEXT: ld1 { v17.b }[15], [x11] +; CHECK-SD-NEXT: add x10, sp, #1344 +; CHECK-SD-NEXT: add x11, sp, #960 +; CHECK-SD-NEXT: ld1 { v3.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x10] +; CHECK-SD-NEXT: ld1 { v16.b }[14], [x11] +; CHECK-SD-NEXT: add x9, sp, #584 +; CHECK-SD-NEXT: sdot v5.4s, v1.16b, v0.16b ; CHECK-SD-NEXT: add x8, sp, #200 -; CHECK-SD-NEXT: add x9, sp, #968 -; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b -; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8] -; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] -; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b -; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b -; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s +; CHECK-SD-NEXT: sdot v4.4s, v17.16b, v6.16b +; CHECK-SD-NEXT: ld1 { v2.b }[15], [x9] +; CHECK-SD-NEXT: add x9, sp, #1352 +; CHECK-SD-NEXT: add x10, sp, #968 +; CHECK-SD-NEXT: ld1 { v3.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x9] +; CHECK-SD-NEXT: ld1 { v16.b }[15], [x10] +; CHECK-SD-NEXT: sdot v5.4s, v3.16b, v2.16b +; CHECK-SD-NEXT: sdot v4.4s, v16.16b, v7.16b +; CHECK-SD-NEXT: add v0.4s, v5.4s, v4.4s ; CHECK-SD-NEXT: addv s0, v0.4s ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-SD-NEXT: ret ; -; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla: +; CHECK-GI-LABEL: test_sdot_v48i8_double: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 ; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: ldr w10, [sp, #80] -; CHECK-GI-NEXT: ldr w11, [sp, #208] +; CHECK-GI-NEXT: ldr w11, [sp, #80] +; CHECK-GI-NEXT: ldr w10, [sp, #208] ; CHECK-GI-NEXT: fmov s0, w0 -; CHECK-GI-NEXT: ldr w9, [sp, #88] -; CHECK-GI-NEXT: ldr w12, [sp, #728] -; CHECK-GI-NEXT: movi v6.16b, #1 -; CHECK-GI-NEXT: fmov s1, w10 -; CHECK-GI-NEXT: fmov s2, w11 -; CHECK-GI-NEXT: ldr w11, [sp, #720] -; CHECK-GI-NEXT: ldr w10, [sp, #216] +; CHECK-GI-NEXT: ldr w8, [sp, #88] +; CHECK-GI-NEXT: ldr w12, [sp, #344] +; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 +; CHECK-GI-NEXT: fmov s1, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #336] +; CHECK-GI-NEXT: fmov s2, w10 +; CHECK-GI-NEXT: ldr w10, [sp, #464] +; CHECK-GI-NEXT: ldr w9, [sp, #216] ; CHECK-GI-NEXT: mov v0.b[1], w1 -; CHECK-GI-NEXT: ldr w13, [sp, #856] ; CHECK-GI-NEXT: fmov s3, w11 -; CHECK-GI-NEXT: ldr w8, [sp, #96] -; CHECK-GI-NEXT: ldr w11, [sp, #224] -; CHECK-GI-NEXT: mov v1.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #848] -; CHECK-GI-NEXT: mov v2.b[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #976] -; CHECK-GI-NEXT: movi v7.2d, #0000000000000000 -; CHECK-GI-NEXT: movi v16.2d, #0000000000000000 -; CHECK-GI-NEXT: fmov s4, w9 +; CHECK-GI-NEXT: ldr w11, [sp, #600] +; CHECK-GI-NEXT: movi v21.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v1.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #592] +; CHECK-GI-NEXT: fmov s4, w10 +; CHECK-GI-NEXT: mov v2.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #472] +; CHECK-GI-NEXT: ldr w10, [sp, #608] ; CHECK-GI-NEXT: mov v3.b[1], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #984] -; CHECK-GI-NEXT: fmov s5, w10 +; CHECK-GI-NEXT: fmov s5, w8 +; CHECK-GI-NEXT: ldr w8, [sp, #96] +; CHECK-GI-NEXT: mov v4.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #224] ; CHECK-GI-NEXT: mov v0.b[2], w2 -; CHECK-GI-NEXT: ldr w10, [sp, #736] ; CHECK-GI-NEXT: mov v1.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #864] -; CHECK-GI-NEXT: mov v2.b[2], w11 -; CHECK-GI-NEXT: mov v4.b[1], w13 -; CHECK-GI-NEXT: ldr w11, [sp, #992] -; CHECK-GI-NEXT: ldr w12, [sp, #776] -; CHECK-GI-NEXT: mov v5.b[1], w9 -; CHECK-GI-NEXT: mov v3.b[2], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #104] -; CHECK-GI-NEXT: ldr w10, [sp, #232] +; CHECK-GI-NEXT: ldr w8, [sp, #352] +; CHECK-GI-NEXT: ldr w12, [sp, #848] +; CHECK-GI-NEXT: mov v2.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #480] +; CHECK-GI-NEXT: mov v5.b[1], w11 +; CHECK-GI-NEXT: mov v3.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #104] +; CHECK-GI-NEXT: ldr w11, [sp, #16] +; CHECK-GI-NEXT: mov v4.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #232] ; CHECK-GI-NEXT: mov v0.b[3], w3 -; CHECK-GI-NEXT: movi v17.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v1.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #872] -; CHECK-GI-NEXT: movi v18.2d, #0000000000000000 -; CHECK-GI-NEXT: mov v4.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #744] -; CHECK-GI-NEXT: mov v2.b[3], w10 -; CHECK-GI-NEXT: mov v5.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1000] -; CHECK-GI-NEXT: ldr w10, [sp, #240] +; CHECK-GI-NEXT: mov v1.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #360] +; CHECK-GI-NEXT: fmov s7, w12 +; CHECK-GI-NEXT: mov v2.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #488] +; CHECK-GI-NEXT: mov v5.b[2], w10 ; CHECK-GI-NEXT: mov v3.b[3], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #112] -; CHECK-GI-NEXT: mov v0.b[4], w4 -; CHECK-GI-NEXT: movi v19.2d, #0000000000000000 -; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 +; CHECK-GI-NEXT: ldr w10, [sp, #616] ; CHECK-GI-NEXT: mov v4.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #752] +; CHECK-GI-NEXT: ldr w9, [sp, #240] +; CHECK-GI-NEXT: mov v0.b[4], w4 ; CHECK-GI-NEXT: mov v1.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #880] -; CHECK-GI-NEXT: mov v5.b[3], w11 -; CHECK-GI-NEXT: mov v2.b[4], w10 -; CHECK-GI-NEXT: mov v3.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #120] -; CHECK-GI-NEXT: ldr w11, [sp, #1008] -; CHECK-GI-NEXT: ldr w10, [sp, #248] +; CHECK-GI-NEXT: ldr w8, [sp, #368] +; CHECK-GI-NEXT: ldr w12, [sp, #1112] +; CHECK-GI-NEXT: mov v2.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #496] +; CHECK-GI-NEXT: mov v5.b[3], w10 +; CHECK-GI-NEXT: mov v3.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #120] +; CHECK-GI-NEXT: ldr w10, [sp, #624] +; CHECK-GI-NEXT: mov v4.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #248] ; CHECK-GI-NEXT: mov v0.b[5], w5 -; CHECK-GI-NEXT: mov v4.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #760] -; CHECK-GI-NEXT: mov v1.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #888] -; CHECK-GI-NEXT: mov v5.b[4], w11 -; CHECK-GI-NEXT: mov v2.b[5], w10 +; CHECK-GI-NEXT: mov v1.b[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #376] +; CHECK-GI-NEXT: movi v22.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #504] +; CHECK-GI-NEXT: mov v5.b[4], w10 ; CHECK-GI-NEXT: mov v3.b[5], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #128] -; CHECK-GI-NEXT: ldr w11, [sp, #1016] -; CHECK-GI-NEXT: ldr w10, [sp, #256] -; CHECK-GI-NEXT: mov v0.b[6], w6 +; CHECK-GI-NEXT: ldr w10, [sp, #632] ; CHECK-GI-NEXT: mov v4.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #768] +; CHECK-GI-NEXT: ldr w9, [sp, #256] +; CHECK-GI-NEXT: mov v0.b[6], w6 ; CHECK-GI-NEXT: mov v1.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #896] -; CHECK-GI-NEXT: mov v5.b[5], w11 -; CHECK-GI-NEXT: mov v2.b[6], w10 -; CHECK-GI-NEXT: mov v3.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #136] -; CHECK-GI-NEXT: ldr w11, [sp, #1024] -; CHECK-GI-NEXT: ldr w10, [sp, #264] +; CHECK-GI-NEXT: ldr w8, [sp, #384] +; CHECK-GI-NEXT: movi v23.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #512] +; CHECK-GI-NEXT: mov v5.b[5], w10 +; CHECK-GI-NEXT: mov v3.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #136] +; CHECK-GI-NEXT: ldr w10, [sp, #640] +; CHECK-GI-NEXT: mov v4.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #264] ; CHECK-GI-NEXT: mov v0.b[7], w7 -; CHECK-GI-NEXT: mov v4.b[6], w8 -; CHECK-GI-NEXT: mov v1.b[7], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #904] -; CHECK-GI-NEXT: mov v5.b[6], w11 -; CHECK-GI-NEXT: mov v2.b[7], w10 -; CHECK-GI-NEXT: ldr w8, [sp, #16] -; CHECK-GI-NEXT: mov v3.b[7], w12 -; CHECK-GI-NEXT: ldr w10, [sp, #144] -; CHECK-GI-NEXT: ldr w12, [sp, #1032] -; CHECK-GI-NEXT: mov v0.b[8], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #784] -; CHECK-GI-NEXT: ldr w11, [sp, #272] +; CHECK-GI-NEXT: mov v1.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #392] +; CHECK-GI-NEXT: movi v24.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #520] +; CHECK-GI-NEXT: mov v5.b[6], w10 +; CHECK-GI-NEXT: mov v3.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #144] +; CHECK-GI-NEXT: ldr w10, [sp, #648] ; CHECK-GI-NEXT: mov v4.b[7], w9 -; CHECK-GI-NEXT: mov v1.b[8], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #912] -; CHECK-GI-NEXT: mov v5.b[7], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #24] -; CHECK-GI-NEXT: ldr w12, [sp, #1040] +; CHECK-GI-NEXT: ldr w9, [sp, #272] +; CHECK-GI-NEXT: mov v0.b[8], w11 +; CHECK-GI-NEXT: mov v1.b[8], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #400] +; CHECK-GI-NEXT: ldr w11, [sp, #24] +; CHECK-GI-NEXT: mov v2.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #528] +; CHECK-GI-NEXT: mov v5.b[7], w10 ; CHECK-GI-NEXT: mov v3.b[8], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #152] -; CHECK-GI-NEXT: mov v2.b[8], w11 -; CHECK-GI-NEXT: mov v0.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #792] -; CHECK-GI-NEXT: ldr w11, [sp, #280] -; CHECK-GI-NEXT: mov v4.b[8], w10 -; CHECK-GI-NEXT: mov v1.b[9], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #920] -; CHECK-GI-NEXT: mov v5.b[8], w12 -; CHECK-GI-NEXT: ldr w8, [sp, #32] -; CHECK-GI-NEXT: ldr w12, [sp, #1048] -; CHECK-GI-NEXT: mov v3.b[9], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #160] -; CHECK-GI-NEXT: mov v2.b[9], w11 -; CHECK-GI-NEXT: mov v0.b[10], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #800] -; CHECK-GI-NEXT: ldr w11, [sp, #288] -; CHECK-GI-NEXT: mov v4.b[9], w10 -; CHECK-GI-NEXT: mov v1.b[10], w9 -; CHECK-GI-NEXT: ldr w10, [sp, #928] -; CHECK-GI-NEXT: mov v5.b[9], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #40] -; CHECK-GI-NEXT: ldr w12, [sp, #1056] +; CHECK-GI-NEXT: ldr w10, [sp, #656] +; CHECK-GI-NEXT: mov v4.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #280] +; CHECK-GI-NEXT: mov v0.b[9], w11 +; CHECK-GI-NEXT: mov v1.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #408] +; CHECK-GI-NEXT: ldr w11, [sp, #32] +; CHECK-GI-NEXT: mov v2.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #536] +; CHECK-GI-NEXT: mov v5.b[8], w10 +; CHECK-GI-NEXT: mov v3.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #160] +; CHECK-GI-NEXT: ldr w10, [sp, #664] +; CHECK-GI-NEXT: mov v4.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #288] +; CHECK-GI-NEXT: mov v0.b[10], w11 +; CHECK-GI-NEXT: mov v1.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #416] +; CHECK-GI-NEXT: ldr w11, [sp, #40] +; CHECK-GI-NEXT: mov v2.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #544] +; CHECK-GI-NEXT: mov v5.b[9], w10 ; CHECK-GI-NEXT: mov v3.b[10], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #168] -; CHECK-GI-NEXT: mov v2.b[10], w11 -; CHECK-GI-NEXT: mov v0.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #808] -; CHECK-GI-NEXT: ldr w11, [sp, #296] -; CHECK-GI-NEXT: mov v4.b[10], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #672] +; CHECK-GI-NEXT: mov v4.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #296] +; CHECK-GI-NEXT: mov v0.b[11], w11 ; CHECK-GI-NEXT: mov v1.b[11], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #936] -; CHECK-GI-NEXT: mov v5.b[10], w12 -; CHECK-GI-NEXT: ldr w8, [sp, #48] -; CHECK-GI-NEXT: ldr w12, [sp, #1064] -; CHECK-GI-NEXT: mov v3.b[11], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #176] -; CHECK-GI-NEXT: mov v2.b[11], w11 -; CHECK-GI-NEXT: mov v0.b[12], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #816] -; CHECK-GI-NEXT: ldr w11, [sp, #304] -; CHECK-GI-NEXT: mov v4.b[11], w10 -; CHECK-GI-NEXT: mov v1.b[12], w9 -; CHECK-GI-NEXT: ldr w10, [sp, #944] -; CHECK-GI-NEXT: mov v5.b[11], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #56] -; CHECK-GI-NEXT: ldr w12, [sp, #1072] +; CHECK-GI-NEXT: ldr w8, [sp, #424] +; CHECK-GI-NEXT: ldr w11, [sp, #48] +; CHECK-GI-NEXT: mov v2.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #552] +; CHECK-GI-NEXT: mov v5.b[10], w10 +; CHECK-GI-NEXT: mov v3.b[11], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #176] +; CHECK-GI-NEXT: ldr w10, [sp, #680] +; CHECK-GI-NEXT: mov v4.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #304] +; CHECK-GI-NEXT: mov v0.b[12], w11 +; CHECK-GI-NEXT: mov v1.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #432] +; CHECK-GI-NEXT: ldr w11, [sp, #56] +; CHECK-GI-NEXT: mov v2.b[12], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #560] +; CHECK-GI-NEXT: mov v5.b[11], w10 ; CHECK-GI-NEXT: mov v3.b[12], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #184] -; CHECK-GI-NEXT: mov v2.b[12], w11 -; CHECK-GI-NEXT: mov v0.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #824] -; CHECK-GI-NEXT: ldr w11, [sp, #312] -; CHECK-GI-NEXT: mov v4.b[12], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #688] +; CHECK-GI-NEXT: mov v4.b[12], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #312] +; CHECK-GI-NEXT: mov v0.b[13], w11 ; CHECK-GI-NEXT: mov v1.b[13], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #952] -; CHECK-GI-NEXT: mov v5.b[12], w12 -; CHECK-GI-NEXT: ldr w8, [sp, #64] -; CHECK-GI-NEXT: ldr w12, [sp, #1080] -; CHECK-GI-NEXT: mov v3.b[13], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #192] -; CHECK-GI-NEXT: mov v2.b[13], w11 -; CHECK-GI-NEXT: mov v0.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #832] -; CHECK-GI-NEXT: ldr w11, [sp, #320] -; CHECK-GI-NEXT: mov v4.b[13], w10 -; CHECK-GI-NEXT: mov v1.b[14], w9 -; CHECK-GI-NEXT: ldr w10, [sp, #960] -; CHECK-GI-NEXT: mov v5.b[13], w12 -; CHECK-GI-NEXT: ldr w9, [sp, #72] -; CHECK-GI-NEXT: ldr w12, [sp, #1088] +; CHECK-GI-NEXT: ldr w8, [sp, #440] +; CHECK-GI-NEXT: ldr w11, [sp, #64] +; CHECK-GI-NEXT: mov v2.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #568] +; CHECK-GI-NEXT: mov v5.b[12], w10 +; CHECK-GI-NEXT: mov v3.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #192] +; CHECK-GI-NEXT: ldr w10, [sp, #696] +; CHECK-GI-NEXT: mov v4.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #320] +; CHECK-GI-NEXT: mov v0.b[14], w11 +; CHECK-GI-NEXT: mov v1.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #448] +; CHECK-GI-NEXT: ldr w11, [sp, #72] +; CHECK-GI-NEXT: mov v2.b[14], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #576] +; CHECK-GI-NEXT: mov v5.b[13], w10 ; CHECK-GI-NEXT: mov v3.b[14], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #200] -; CHECK-GI-NEXT: mov v2.b[14], w11 -; CHECK-GI-NEXT: mov v0.b[15], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #840] -; CHECK-GI-NEXT: ldr w11, [sp, #328] -; CHECK-GI-NEXT: mov v4.b[14], w10 -; CHECK-GI-NEXT: mov v1.b[15], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #968] -; CHECK-GI-NEXT: mov v5.b[14], w12 -; CHECK-GI-NEXT: ldr w10, [sp, #1096] -; CHECK-GI-NEXT: mov v3.b[15], w9 -; CHECK-GI-NEXT: mov v2.b[15], w11 -; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b -; CHECK-GI-NEXT: mov v4.b[15], w8 -; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b +; CHECK-GI-NEXT: ldr w8, [sp, #720] +; CHECK-GI-NEXT: ldr w10, [sp, #704] +; CHECK-GI-NEXT: mov v4.b[14], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #728] +; CHECK-GI-NEXT: mov v0.b[15], w11 +; CHECK-GI-NEXT: fmov s6, w8 +; CHECK-GI-NEXT: ldr w8, [sp, #328] +; CHECK-GI-NEXT: ldr w11, [sp, #456] +; CHECK-GI-NEXT: mov v5.b[14], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #200] +; CHECK-GI-NEXT: movi v25.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v2.b[15], w8 +; CHECK-GI-NEXT: mov v3.b[15], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #736] +; CHECK-GI-NEXT: mov v6.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #584] +; CHECK-GI-NEXT: ldr w8, [sp, #856] +; CHECK-GI-NEXT: mov v1.b[15], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #712] +; CHECK-GI-NEXT: mov v4.b[15], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #976] +; CHECK-GI-NEXT: mov v7.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1232] ; CHECK-GI-NEXT: mov v5.b[15], w10 -; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b -; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b -; CHECK-GI-NEXT: addv s0, v7.4s -; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b -; CHECK-GI-NEXT: addv s1, v16.4s -; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b -; CHECK-GI-NEXT: addv s2, v17.4s -; CHECK-GI-NEXT: addv s4, v20.4s +; CHECK-GI-NEXT: ldr w10, [sp, #984] +; CHECK-GI-NEXT: mov v6.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1104] +; CHECK-GI-NEXT: fmov s16, w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1360] +; CHECK-GI-NEXT: fmov s18, w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1368] +; CHECK-GI-NEXT: fmov s17, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1240] +; CHECK-GI-NEXT: sdot v20.4s, v0.16b, v3.16b +; CHECK-GI-NEXT: mov v16.b[1], w10 +; CHECK-GI-NEXT: fmov s19, w9 +; CHECK-GI-NEXT: ldr w10, [sp, #864] +; CHECK-GI-NEXT: mov v18.b[1], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #992] +; CHECK-GI-NEXT: ldr w9, [sp, #1120] +; CHECK-GI-NEXT: mov v17.b[1], w12 +; CHECK-GI-NEXT: mov v7.b[2], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1248] +; CHECK-GI-NEXT: mov v19.b[1], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #744] +; CHECK-GI-NEXT: sdot v21.4s, v1.16b, v4.16b +; CHECK-GI-NEXT: mov v16.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #872] +; CHECK-GI-NEXT: addv s0, v20.4s +; CHECK-GI-NEXT: mov v6.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1000] +; CHECK-GI-NEXT: mov v18.b[2], w10 +; CHECK-GI-NEXT: mov v17.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1376] +; CHECK-GI-NEXT: ldr w10, [sp, #1128] +; CHECK-GI-NEXT: mov v7.b[3], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #880] +; CHECK-GI-NEXT: addv s1, v21.4s +; CHECK-GI-NEXT: mov v19.b[2], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #752] +; CHECK-GI-NEXT: mov v16.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1256] +; CHECK-GI-NEXT: sdot v25.4s, v2.16b, v5.16b +; CHECK-GI-NEXT: mov v17.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1384] +; CHECK-GI-NEXT: mov v6.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1008] +; CHECK-GI-NEXT: mov v18.b[3], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1136] +; CHECK-GI-NEXT: mov v19.b[3], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #760] +; CHECK-GI-NEXT: mov v7.b[4], w11 +; CHECK-GI-NEXT: mov v16.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1264] +; CHECK-GI-NEXT: ldr w11, [sp, #888] +; CHECK-GI-NEXT: mov v17.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1392] +; CHECK-GI-NEXT: mov v6.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1016] +; CHECK-GI-NEXT: mov v18.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1144] +; CHECK-GI-NEXT: mov v19.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #768] +; CHECK-GI-NEXT: mov v7.b[5], w11 +; CHECK-GI-NEXT: mov v16.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1272] +; CHECK-GI-NEXT: ldr w11, [sp, #896] +; CHECK-GI-NEXT: mov v17.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1400] +; CHECK-GI-NEXT: mov v6.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1024] +; CHECK-GI-NEXT: mov v18.b[5], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1152] +; CHECK-GI-NEXT: mov v19.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #776] +; CHECK-GI-NEXT: mov v7.b[6], w11 +; CHECK-GI-NEXT: mov v16.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1280] +; CHECK-GI-NEXT: ldr w11, [sp, #904] +; CHECK-GI-NEXT: mov v17.b[6], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1408] +; CHECK-GI-NEXT: mov v6.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1032] +; CHECK-GI-NEXT: mov v18.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1160] +; CHECK-GI-NEXT: mov v19.b[6], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #784] +; CHECK-GI-NEXT: mov v7.b[7], w11 +; CHECK-GI-NEXT: mov v16.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1288] +; CHECK-GI-NEXT: ldr w11, [sp, #912] +; CHECK-GI-NEXT: mov v17.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1416] +; CHECK-GI-NEXT: mov v6.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1040] +; CHECK-GI-NEXT: mov v18.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1168] +; CHECK-GI-NEXT: mov v19.b[7], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #792] +; CHECK-GI-NEXT: mov v7.b[8], w11 +; CHECK-GI-NEXT: mov v16.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1296] +; CHECK-GI-NEXT: ldr w11, [sp, #920] +; CHECK-GI-NEXT: mov v17.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1424] +; CHECK-GI-NEXT: mov v6.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1048] +; CHECK-GI-NEXT: mov v18.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1176] +; CHECK-GI-NEXT: mov v19.b[8], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #800] +; CHECK-GI-NEXT: mov v7.b[9], w11 +; CHECK-GI-NEXT: mov v16.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1304] +; CHECK-GI-NEXT: ldr w11, [sp, #928] +; CHECK-GI-NEXT: mov v17.b[9], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1432] +; CHECK-GI-NEXT: mov v6.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1056] +; CHECK-GI-NEXT: mov v18.b[9], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1184] +; CHECK-GI-NEXT: mov v19.b[9], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #808] +; CHECK-GI-NEXT: mov v7.b[10], w11 +; CHECK-GI-NEXT: mov v16.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1312] +; CHECK-GI-NEXT: ldr w11, [sp, #936] +; CHECK-GI-NEXT: mov v17.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1440] +; CHECK-GI-NEXT: mov v6.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1064] +; CHECK-GI-NEXT: mov v18.b[10], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1192] +; CHECK-GI-NEXT: mov v19.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #816] +; CHECK-GI-NEXT: mov v7.b[11], w11 +; CHECK-GI-NEXT: mov v16.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1320] +; CHECK-GI-NEXT: ldr w11, [sp, #944] +; CHECK-GI-NEXT: mov v17.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1448] +; CHECK-GI-NEXT: mov v6.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1072] +; CHECK-GI-NEXT: mov v18.b[11], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1200] +; CHECK-GI-NEXT: mov v19.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #824] +; CHECK-GI-NEXT: mov v7.b[12], w11 +; CHECK-GI-NEXT: mov v16.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1328] +; CHECK-GI-NEXT: ldr w11, [sp, #952] +; CHECK-GI-NEXT: mov v17.b[12], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1456] +; CHECK-GI-NEXT: mov v6.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1080] +; CHECK-GI-NEXT: mov v18.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1208] +; CHECK-GI-NEXT: mov v19.b[12], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #832] +; CHECK-GI-NEXT: mov v7.b[13], w11 +; CHECK-GI-NEXT: mov v16.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1336] +; CHECK-GI-NEXT: ldr w11, [sp, #960] +; CHECK-GI-NEXT: mov v17.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1464] +; CHECK-GI-NEXT: mov v6.b[14], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1088] +; CHECK-GI-NEXT: mov v18.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #1216] +; CHECK-GI-NEXT: mov v19.b[13], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #840] +; CHECK-GI-NEXT: mov v7.b[14], w11 +; CHECK-GI-NEXT: mov v16.b[14], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #1344] +; CHECK-GI-NEXT: ldr w11, [sp, #968] +; CHECK-GI-NEXT: mov v17.b[14], w9 +; CHECK-GI-NEXT: mov v6.b[15], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1096] +; CHECK-GI-NEXT: mov v18.b[14], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #1472] +; CHECK-GI-NEXT: ldr w10, [sp, #1224] +; CHECK-GI-NEXT: mov v7.b[15], w11 +; CHECK-GI-NEXT: addv s4, v25.4s +; CHECK-GI-NEXT: mov v16.b[15], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #1352] +; CHECK-GI-NEXT: mov v19.b[14], w9 +; CHECK-GI-NEXT: mov v17.b[15], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #1480] +; CHECK-GI-NEXT: mov v18.b[15], w8 ; CHECK-GI-NEXT: fmov w8, s0 +; CHECK-GI-NEXT: fmov w11, s4 +; CHECK-GI-NEXT: mov v19.b[15], w9 ; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: addv s3, v18.4s -; CHECK-GI-NEXT: addv s5, v19.4s -; CHECK-GI-NEXT: fmov w10, s2 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w9, s4 -; CHECK-GI-NEXT: fmov w11, s3 +; CHECK-GI-NEXT: sdot v22.4s, v6.16b, v17.16b +; CHECK-GI-NEXT: sdot v23.4s, v7.16b, v18.16b ; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s5 -; CHECK-GI-NEXT: add w9, w10, w11 +; CHECK-GI-NEXT: sdot v24.4s, v16.16b, v19.16b +; CHECK-GI-NEXT: add w8, w8, w11 +; CHECK-GI-NEXT: addv s2, v22.4s +; CHECK-GI-NEXT: addv s3, v23.4s +; CHECK-GI-NEXT: addv s5, v24.4s +; CHECK-GI-NEXT: fmov w9, s2 +; CHECK-GI-NEXT: fmov w10, s3 +; CHECK-GI-NEXT: add w9, w9, w10 +; CHECK-GI-NEXT: fmov w10, s5 +; CHECK-GI-NEXT: add w9, w9, w10 ; CHECK-GI-NEXT: add w0, w8, w9 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-GI-NEXT: ret entry: %az = sext <48 x i8> %a to <48 x i32> - %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az) + %bz = sext <48 x i8> %b to <48 x i32> + %m1 = mul nuw nsw <48 x i32> %az, %bz + %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1) %cz = sext <48 x i8> %c to <48 x i32> - %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz) + %dz = sext <48 x i8> %d to <48 x i32> + %m2 = mul nuw nsw <48 x i32> %cz, %dz + %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2) %x = add i32 %r1, %r2 ret i32 %x } -define i32 @test_usdot_v48i8(ptr nocapture readonly %a, ptr nocapture readonly %b, i32 %sum) { -; CHECK-SD-LABEL: test_usdot_v48i8: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: movi v0.2d, #0000000000000000 -; CHECK-SD-NEXT: movi v1.2d, #0000000000000000 -; CHECK-SD-NEXT: ldr q3, [x0, #32] -; CHECK-SD-NEXT: ldp q7, q2, [x0] -; CHECK-SD-NEXT: ldr q4, [x1] -; CHECK-SD-NEXT: ldr q5, [x1, #32] -; CHECK-SD-NEXT: movi v6.2d, #0000000000000000 -; CHECK-SD-NEXT: usdot v0.4s, v3.16b, v5.16b -; CHECK-SD-NEXT: usdot v1.4s, v7.16b, v4.16b -; CHECK-SD-NEXT: ldr q3, [x1, #16] -; CHECK-SD-NEXT: usdot v6.4s, v2.16b, v3.16b -; CHECK-SD-NEXT: add v0.4s, v1.4s, v0.4s -; CHECK-SD-NEXT: add v0.4s, v0.4s, v6.4s -; CHECK-SD-NEXT: addv s0, v0.4s -; CHECK-SD-NEXT: fmov w8, s0 -; CHECK-SD-NEXT: add w0, w8, w2 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_usdot_v48i8: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: ldp q0, q2, [x1] -; CHECK-GI-NEXT: ldr q7, [x1, #32] -; CHECK-GI-NEXT: ldp q1, q4, [x0] -; CHECK-GI-NEXT: ldr q6, [x0, #32] -; CHECK-GI-NEXT: sshll v20.8h, v7.8b, #0 -; CHECK-GI-NEXT: sshll v3.8h, v0.8b, #0 -; CHECK-GI-NEXT: sshll2 v18.8h, v0.16b, #0 -; CHECK-GI-NEXT: sshll v19.8h, v2.8b, #0 -; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0 -; CHECK-GI-NEXT: sshll2 v0.8h, v7.16b, #0 -; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0 -; CHECK-GI-NEXT: ushll v7.8h, v4.8b, #0 -; CHECK-GI-NEXT: sshll2 v2.8h, v2.16b, #0 -; CHECK-GI-NEXT: ushll2 v4.8h, v4.16b, #0 -; CHECK-GI-NEXT: sshll v16.4s, v3.4h, #0 -; CHECK-GI-NEXT: sshll2 v3.4s, v3.8h, #0 -; CHECK-GI-NEXT: sshll v21.4s, v19.4h, #0 -; CHECK-GI-NEXT: ushll v17.4s, v5.4h, #0 -; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 -; CHECK-GI-NEXT: ushll v25.4s, v1.4h, #0 -; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0 -; CHECK-GI-NEXT: sshll v22.4s, v2.4h, #0 -; CHECK-GI-NEXT: sshll2 v2.4s, v2.8h, #0 -; CHECK-GI-NEXT: ushll v27.4s, v4.4h, #0 -; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0 -; CHECK-GI-NEXT: mul v3.4s, v3.4s, v5.4s -; CHECK-GI-NEXT: mul v16.4s, v16.4s, v17.4s -; CHECK-GI-NEXT: sshll v17.4s, v18.4h, #0 -; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0 -; CHECK-GI-NEXT: ushll v5.8h, v6.8b, #0 -; CHECK-GI-NEXT: ushll2 v6.8h, v6.16b, #0 -; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0 -; CHECK-GI-NEXT: sshll v23.4s, v20.4h, #0 -; CHECK-GI-NEXT: mul v2.4s, v2.4s, v4.4s -; CHECK-GI-NEXT: sshll v24.4s, v0.4h, #0 -; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0 -; CHECK-GI-NEXT: addv s3, v3.4s -; CHECK-GI-NEXT: addv s16, v16.4s -; CHECK-GI-NEXT: mul v1.4s, v18.4s, v1.4s -; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0 -; CHECK-GI-NEXT: ushll v18.4s, v6.4h, #0 -; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: fmov w9, s3 -; CHECK-GI-NEXT: ushll2 v3.4s, v7.8h, #0 -; CHECK-GI-NEXT: mul v7.4s, v17.4s, v25.4s -; CHECK-GI-NEXT: mul v17.4s, v21.4s, v26.4s -; CHECK-GI-NEXT: fmov w8, s16 -; CHECK-GI-NEXT: ushll v16.4s, v5.4h, #0 -; CHECK-GI-NEXT: mul v21.4s, v22.4s, v27.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 -; CHECK-GI-NEXT: mul v3.4s, v19.4s, v3.4s -; CHECK-GI-NEXT: mul v0.4s, v0.4s, v4.4s -; CHECK-GI-NEXT: fmov w12, s2 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: mul v6.4s, v23.4s, v16.4s -; CHECK-GI-NEXT: mul v16.4s, v24.4s, v18.4s -; CHECK-GI-NEXT: addv s17, v17.4s -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: mul v5.4s, v20.4s, v5.4s -; CHECK-GI-NEXT: addv s4, v21.4s -; CHECK-GI-NEXT: addv s7, v7.4s -; CHECK-GI-NEXT: addv s1, v3.4s -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: fmov w11, s17 -; CHECK-GI-NEXT: addv s3, v6.4s -; CHECK-GI-NEXT: fmov w9, s7 -; CHECK-GI-NEXT: addv s2, v5.4s -; CHECK-GI-NEXT: fmov w13, s1 -; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s4 -; CHECK-GI-NEXT: addv s4, v16.4s -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w9, w10, w13 -; CHECK-GI-NEXT: fmov w13, s0 -; CHECK-GI-NEXT: add w11, w11, w12 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: fmov w12, s3 -; CHECK-GI-NEXT: add w10, w11, w12 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: fmov w12, s4 -; CHECK-GI-NEXT: add w9, w10, w11 -; CHECK-GI-NEXT: add w10, w12, w13 -; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w9, w10, w2 -; CHECK-GI-NEXT: add w0, w8, w9 -; CHECK-GI-NEXT: ret -entry: - %0 = load <48 x i8>, ptr %a - %1 = zext <48 x i8> %0 to <48 x i32> - %2 = load <48 x i8>, ptr %b - %3 = sext <48 x i8> %2 to <48 x i32> - %4 = mul nsw <48 x i32> %3, %1 - %5 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %4) - %op.extra = add nsw i32 %5, %sum - ret i32 %op.extra -} - -define i32 @test_usdot_v48i8_double(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { -; CHECK-SD-LABEL: test_usdot_v48i8_double: +define i32 @test_sdot_v48i8_double_nomla(<48 x i8> %a, <48 x i8> %b, <48 x i8> %c, <48 x i8> %d) { +; CHECK-SD-LABEL: test_sdot_v48i8_double_nomla: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-SD-NEXT: .cfi_def_cfa_offset 16 ; CHECK-SD-NEXT: .cfi_offset w29, -16 -; CHECK-SD-NEXT: ldr b2, [sp, #592] -; CHECK-SD-NEXT: add x8, sp, #600 ; CHECK-SD-NEXT: ldr b5, [sp, #208] -; CHECK-SD-NEXT: add x9, sp, #616 -; CHECK-SD-NEXT: ldr b0, [sp, #336] -; CHECK-SD-NEXT: add x10, sp, #624 -; CHECK-SD-NEXT: ld1 { v2.b }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #216 -; CHECK-SD-NEXT: add x11, sp, #240 +; CHECK-SD-NEXT: fmov s0, w0 +; CHECK-SD-NEXT: ldr b4, [sp, #976] +; CHECK-SD-NEXT: add x9, sp, #984 +; CHECK-SD-NEXT: add x12, sp, #328 ; CHECK-SD-NEXT: ld1 { v5.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #608 -; CHECK-SD-NEXT: ldr b4, [sp, #464] -; CHECK-SD-NEXT: ldr b6, [sp, #80] -; CHECK-SD-NEXT: ldr b7, [sp, #1360] -; CHECK-SD-NEXT: ldr b16, [sp, #976] -; CHECK-SD-NEXT: ld1 { v2.b }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #224 -; CHECK-SD-NEXT: fmov s1, w0 -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #344 +; CHECK-SD-NEXT: movi v1.16b, #1 +; CHECK-SD-NEXT: mov v0.b[1], w1 +; CHECK-SD-NEXT: ld1 { v4.b }[1], [x9] ; CHECK-SD-NEXT: movi v3.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v0.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #232 -; CHECK-SD-NEXT: ldr b18, [sp, #720] -; CHECK-SD-NEXT: ld1 { v2.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #632 -; CHECK-SD-NEXT: mov v1.b[1], w1 -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] -; CHECK-SD-NEXT: add x8, sp, #472 -; CHECK-SD-NEXT: add x12, sp, #856 -; CHECK-SD-NEXT: ld1 { v4.b }[1], [x8] -; CHECK-SD-NEXT: add x8, sp, #640 -; CHECK-SD-NEXT: add x13, sp, #864 -; CHECK-SD-NEXT: ld1 { v2.b }[4], [x10] -; CHECK-SD-NEXT: add x10, sp, #88 -; CHECK-SD-NEXT: movi v17.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x11] -; CHECK-SD-NEXT: ld1 { v6.b }[1], [x10] -; CHECK-SD-NEXT: add x10, sp, #248 -; CHECK-SD-NEXT: add x11, sp, #272 -; CHECK-SD-NEXT: mov v1.b[2], w2 -; CHECK-SD-NEXT: ld1 { v2.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #1368 -; CHECK-SD-NEXT: ld1 { v7.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #984 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[1], [x9] -; CHECK-SD-NEXT: add x9, sp, #352 -; CHECK-SD-NEXT: add x10, sp, #648 -; CHECK-SD-NEXT: ld1 { v2.b }[6], [x8] -; CHECK-SD-NEXT: ld1 { v0.b }[2], [x9] -; CHECK-SD-NEXT: add x9, sp, #256 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9] -; CHECK-SD-NEXT: add x8, sp, #480 -; CHECK-SD-NEXT: add x9, sp, #96 -; CHECK-SD-NEXT: ld1 { v4.b }[2], [x8] -; CHECK-SD-NEXT: add x8, sp, #656 -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x9] -; CHECK-SD-NEXT: ld1 { v2.b }[7], [x10] -; CHECK-SD-NEXT: add x9, sp, #264 -; CHECK-SD-NEXT: add x10, sp, #1376 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9] -; CHECK-SD-NEXT: add x9, sp, #664 -; CHECK-SD-NEXT: ld1 { v7.b }[2], [x10] -; CHECK-SD-NEXT: add x10, sp, #992 -; CHECK-SD-NEXT: mov v1.b[3], w3 -; CHECK-SD-NEXT: ld1 { v2.b }[8], [x8] -; CHECK-SD-NEXT: add x8, sp, #360 -; CHECK-SD-NEXT: ld1 { v16.b }[2], [x10] -; CHECK-SD-NEXT: ld1 { v0.b }[3], [x8] -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x11] -; CHECK-SD-NEXT: add x11, sp, #488 -; CHECK-SD-NEXT: add x10, sp, #672 -; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11] -; CHECK-SD-NEXT: add x11, sp, #280 -; CHECK-SD-NEXT: ld1 { v2.b }[9], [x9] -; CHECK-SD-NEXT: add x9, sp, #104 -; CHECK-SD-NEXT: add x8, sp, #680 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #368 -; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11] -; CHECK-SD-NEXT: ld1 { v0.b }[4], [x9] -; CHECK-SD-NEXT: add x9, sp, #1384 -; CHECK-SD-NEXT: add x11, sp, #688 -; CHECK-SD-NEXT: ld1 { v2.b }[10], [x10] -; CHECK-SD-NEXT: add x10, sp, #376 -; CHECK-SD-NEXT: ld1 { v7.b }[3], [x9] -; CHECK-SD-NEXT: add x9, sp, #288 -; CHECK-SD-NEXT: mov v1.b[4], w4 -; CHECK-SD-NEXT: ld1 { v0.b }[5], [x10] -; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] -; CHECK-SD-NEXT: add x10, sp, #296 -; CHECK-SD-NEXT: ld1 { v2.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #496 -; CHECK-SD-NEXT: add x9, sp, #1000 -; CHECK-SD-NEXT: ld1 { v4.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #384 -; CHECK-SD-NEXT: ld1 { v16.b }[3], [x9] -; CHECK-SD-NEXT: ld1 { v0.b }[6], [x8] -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x10] -; CHECK-SD-NEXT: add x8, sp, #112 -; CHECK-SD-NEXT: ld1 { v2.b }[12], [x11] -; CHECK-SD-NEXT: add x11, sp, #392 -; CHECK-SD-NEXT: add x9, sp, #696 -; CHECK-SD-NEXT: ld1 { v6.b }[4], [x8] -; CHECK-SD-NEXT: add x8, sp, #304 -; CHECK-SD-NEXT: add x10, sp, #704 -; CHECK-SD-NEXT: ld1 { v0.b }[7], [x11] -; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] -; CHECK-SD-NEXT: add x11, sp, #312 -; CHECK-SD-NEXT: ld1 { v2.b }[13], [x9] -; CHECK-SD-NEXT: add x9, sp, #504 -; CHECK-SD-NEXT: add x8, sp, #1392 -; CHECK-SD-NEXT: ld1 { v4.b }[5], [x9] -; CHECK-SD-NEXT: add x9, sp, #400 -; CHECK-SD-NEXT: ld1 { v7.b }[4], [x8] -; CHECK-SD-NEXT: ld1 { v0.b }[8], [x9] -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x11] -; CHECK-SD-NEXT: add x11, sp, #320 -; CHECK-SD-NEXT: ld1 { v2.b }[14], [x10] -; CHECK-SD-NEXT: add x10, sp, #120 -; CHECK-SD-NEXT: add x9, sp, #328 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x10] -; CHECK-SD-NEXT: add x10, sp, #408 -; CHECK-SD-NEXT: add x8, sp, #712 -; CHECK-SD-NEXT: ld1 { v0.b }[9], [x10] -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x11] -; CHECK-SD-NEXT: add x11, sp, #416 -; CHECK-SD-NEXT: add x10, sp, #512 -; CHECK-SD-NEXT: ld1 { v2.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #128 -; CHECK-SD-NEXT: ld1 { v4.b }[6], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #1016 -; CHECK-SD-NEXT: ld1 { v0.b }[10], [x11] -; CHECK-SD-NEXT: add x11, sp, #1400 -; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #1008 -; CHECK-SD-NEXT: ld1 { v7.b }[5], [x11] -; CHECK-SD-NEXT: add x11, sp, #1408 -; CHECK-SD-NEXT: ld1 { v16.b }[4], [x9] -; CHECK-SD-NEXT: add x9, sp, #520 -; CHECK-SD-NEXT: add x10, sp, #424 -; CHECK-SD-NEXT: ld1 { v4.b }[7], [x9] -; CHECK-SD-NEXT: add x9, sp, #136 -; CHECK-SD-NEXT: ld1 { v0.b }[11], [x10] -; CHECK-SD-NEXT: ld1 { v7.b }[6], [x11] -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x9] -; CHECK-SD-NEXT: add x10, sp, #1416 -; CHECK-SD-NEXT: ld1 { v16.b }[5], [x8] -; CHECK-SD-NEXT: add x8, sp, #528 -; CHECK-SD-NEXT: add x9, sp, #144 -; CHECK-SD-NEXT: ld1 { v4.b }[8], [x8] -; CHECK-SD-NEXT: add x8, sp, #1024 -; CHECK-SD-NEXT: add x11, sp, #1432 -; CHECK-SD-NEXT: ld1 { v7.b }[7], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x9] -; CHECK-SD-NEXT: add x10, sp, #1424 -; CHECK-SD-NEXT: ld1 { v16.b }[6], [x8] -; CHECK-SD-NEXT: add x8, sp, #536 -; CHECK-SD-NEXT: add x9, sp, #152 -; CHECK-SD-NEXT: ld1 { v4.b }[9], [x8] -; CHECK-SD-NEXT: add x8, sp, #1032 -; CHECK-SD-NEXT: usdot v3.4s, v5.16b, v2.16b -; CHECK-SD-NEXT: ld1 { v7.b }[8], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] -; CHECK-SD-NEXT: add x10, sp, #160 -; CHECK-SD-NEXT: ld1 { v16.b }[7], [x8] -; CHECK-SD-NEXT: add x8, sp, #544 -; CHECK-SD-NEXT: add x9, sp, #432 -; CHECK-SD-NEXT: ld1 { v4.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #1040 -; CHECK-SD-NEXT: ld1 { v0.b }[12], [x9] -; CHECK-SD-NEXT: ld1 { v7.b }[9], [x11] -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x10] -; CHECK-SD-NEXT: add x11, sp, #1440 -; CHECK-SD-NEXT: ld1 { v16.b }[8], [x8] -; CHECK-SD-NEXT: add x8, sp, #552 -; CHECK-SD-NEXT: add x10, sp, #168 -; CHECK-SD-NEXT: ld1 { v4.b }[11], [x8] -; CHECK-SD-NEXT: add x8, sp, #1048 -; CHECK-SD-NEXT: add x9, sp, #176 -; CHECK-SD-NEXT: ld1 { v7.b }[10], [x11] -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x10] -; CHECK-SD-NEXT: add x10, sp, #1448 -; CHECK-SD-NEXT: ld1 { v16.b }[9], [x8] -; CHECK-SD-NEXT: add x8, sp, #560 -; CHECK-SD-NEXT: add x11, sp, #576 -; CHECK-SD-NEXT: ld1 { v4.b }[12], [x8] -; CHECK-SD-NEXT: add x8, sp, #1056 -; CHECK-SD-NEXT: ldr b5, [sp, #1104] -; CHECK-SD-NEXT: ld1 { v7.b }[11], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[12], [x9] -; CHECK-SD-NEXT: add x10, sp, #1456 -; CHECK-SD-NEXT: ld1 { v16.b }[10], [x8] -; CHECK-SD-NEXT: add x8, sp, #568 -; CHECK-SD-NEXT: add x9, sp, #184 -; CHECK-SD-NEXT: ld1 { v4.b }[13], [x8] -; CHECK-SD-NEXT: add x8, sp, #1064 -; CHECK-SD-NEXT: mov v1.b[5], w5 -; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] -; CHECK-SD-NEXT: add x9, sp, #1072 -; CHECK-SD-NEXT: ld1 { v16.b }[11], [x8] -; CHECK-SD-NEXT: add x10, sp, #192 +; CHECK-SD-NEXT: add x11, sp, #992 +; CHECK-SD-NEXT: ldr b6, [sp, #720] +; CHECK-SD-NEXT: ldr b7, [sp, #80] +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x8] +; CHECK-SD-NEXT: add x8, sp, #232 +; CHECK-SD-NEXT: add x13, sp, #88 +; CHECK-SD-NEXT: ld1 { v4.b }[2], [x11] +; CHECK-SD-NEXT: ld1 { v7.b }[1], [x13] +; CHECK-SD-NEXT: add x13, sp, #856 +; CHECK-SD-NEXT: mov v0.b[2], w2 +; CHECK-SD-NEXT: add x14, sp, #1008 +; CHECK-SD-NEXT: add x15, sp, #872 +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x8] +; CHECK-SD-NEXT: add x8, sp, #240 +; CHECK-SD-NEXT: add x16, sp, #888 +; CHECK-SD-NEXT: add x10, sp, #16 +; CHECK-SD-NEXT: add x9, sp, #24 +; CHECK-SD-NEXT: add x11, sp, #40 ; CHECK-SD-NEXT: movi v2.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v4.b }[14], [x11] -; CHECK-SD-NEXT: add x11, sp, #1464 -; CHECK-SD-NEXT: add x8, sp, #440 -; CHECK-SD-NEXT: ld1 { v7.b }[13], [x11] -; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10] -; CHECK-SD-NEXT: add x11, sp, #1472 -; CHECK-SD-NEXT: ld1 { v16.b }[12], [x9] -; CHECK-SD-NEXT: add x9, sp, #584 -; CHECK-SD-NEXT: add x10, sp, #200 -; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #1080 -; CHECK-SD-NEXT: mov v1.b[6], w6 -; CHECK-SD-NEXT: ld1 { v7.b }[14], [x11] -; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10] -; CHECK-SD-NEXT: add x10, sp, #1112 -; CHECK-SD-NEXT: ld1 { v16.b }[13], [x9] -; CHECK-SD-NEXT: add x11, sp, #1480 -; CHECK-SD-NEXT: ld1 { v5.b }[1], [x10] -; CHECK-SD-NEXT: add x9, sp, #1088 -; CHECK-SD-NEXT: add x10, sp, #1120 -; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8] -; CHECK-SD-NEXT: ld1 { v7.b }[15], [x11] -; CHECK-SD-NEXT: add x11, sp, #728 -; CHECK-SD-NEXT: usdot v2.4s, v6.16b, v4.16b -; CHECK-SD-NEXT: ld1 { v18.b }[1], [x11] -; CHECK-SD-NEXT: ld1 { v16.b }[14], [x9] -; CHECK-SD-NEXT: ld1 { v5.b }[2], [x10] -; CHECK-SD-NEXT: add x10, sp, #736 -; CHECK-SD-NEXT: add x9, sp, #1096 -; CHECK-SD-NEXT: ldr b4, [sp, #1232] -; CHECK-SD-NEXT: ldr b6, [sp, #848] -; CHECK-SD-NEXT: add x11, sp, #1240 -; CHECK-SD-NEXT: mov v1.b[7], w7 -; CHECK-SD-NEXT: ld1 { v18.b }[2], [x10] -; CHECK-SD-NEXT: ld1 { v16.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #1128 -; CHECK-SD-NEXT: ld1 { v5.b }[3], [x9] -; CHECK-SD-NEXT: add x10, sp, #744 -; CHECK-SD-NEXT: ld1 { v4.b }[1], [x11] +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x8] +; CHECK-SD-NEXT: add x8, sp, #248 +; CHECK-SD-NEXT: mov v0.b[3], w3 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x8] +; CHECK-SD-NEXT: add x8, sp, #256 +; CHECK-SD-NEXT: mov v0.b[4], w4 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x8] +; CHECK-SD-NEXT: add x8, sp, #264 +; CHECK-SD-NEXT: mov v0.b[5], w5 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x8] +; CHECK-SD-NEXT: add x8, sp, #272 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x8] +; CHECK-SD-NEXT: add x8, sp, #280 +; CHECK-SD-NEXT: mov v0.b[6], w6 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x8] +; CHECK-SD-NEXT: add x8, sp, #288 +; CHECK-SD-NEXT: mov v0.b[7], w7 +; CHECK-SD-NEXT: ld1 { v5.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #296 +; CHECK-SD-NEXT: ld1 { v0.b }[8], [x10] +; CHECK-SD-NEXT: add x10, sp, #128 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #304 +; CHECK-SD-NEXT: ld1 { v0.b }[9], [x9] +; CHECK-SD-NEXT: add x9, sp, #136 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #312 +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #320 +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] +; CHECK-SD-NEXT: add x8, sp, #32 +; CHECK-SD-NEXT: ld1 { v0.b }[10], [x8] +; CHECK-SD-NEXT: add x8, sp, #144 +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x12] +; CHECK-SD-NEXT: add x12, sp, #728 ; CHECK-SD-NEXT: ld1 { v6.b }[1], [x12] -; CHECK-SD-NEXT: add x9, sp, #1136 -; CHECK-SD-NEXT: add x12, sp, #1248 -; CHECK-SD-NEXT: ld1 { v18.b }[3], [x10] -; CHECK-SD-NEXT: add x11, sp, #752 -; CHECK-SD-NEXT: add x10, sp, #16 -; CHECK-SD-NEXT: ld1 { v5.b }[4], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[2], [x12] -; CHECK-SD-NEXT: add x9, sp, #1144 -; CHECK-SD-NEXT: ld1 { v6.b }[2], [x13] -; CHECK-SD-NEXT: add x12, sp, #872 -; CHECK-SD-NEXT: ld1 { v1.b }[8], [x10] -; CHECK-SD-NEXT: ld1 { v18.b }[4], [x11] -; CHECK-SD-NEXT: add x11, sp, #1256 -; CHECK-SD-NEXT: add x10, sp, #760 -; CHECK-SD-NEXT: ld1 { v5.b }[5], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[3], [x11] -; CHECK-SD-NEXT: add x9, sp, #1152 -; CHECK-SD-NEXT: ld1 { v6.b }[3], [x12] -; CHECK-SD-NEXT: add x12, sp, #1264 -; CHECK-SD-NEXT: add x13, sp, #880 -; CHECK-SD-NEXT: ld1 { v18.b }[5], [x10] -; CHECK-SD-NEXT: add x11, sp, #768 -; CHECK-SD-NEXT: add x10, sp, #24 -; CHECK-SD-NEXT: ld1 { v5.b }[6], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[4], [x12] -; CHECK-SD-NEXT: add x9, sp, #1160 +; CHECK-SD-NEXT: add x12, sp, #1000 +; CHECK-SD-NEXT: ld1 { v0.b }[11], [x11] +; CHECK-SD-NEXT: ld1 { v4.b }[3], [x12] +; CHECK-SD-NEXT: add x12, sp, #736 +; CHECK-SD-NEXT: add x11, sp, #920 +; CHECK-SD-NEXT: sdot v3.4s, v5.16b, v1.16b +; CHECK-SD-NEXT: ldr b5, [sp, #848] +; CHECK-SD-NEXT: ld1 { v6.b }[2], [x12] +; CHECK-SD-NEXT: add x12, sp, #48 +; CHECK-SD-NEXT: ld1 { v5.b }[1], [x13] +; CHECK-SD-NEXT: add x13, sp, #744 +; CHECK-SD-NEXT: ld1 { v4.b }[4], [x14] +; CHECK-SD-NEXT: add x14, sp, #96 +; CHECK-SD-NEXT: ld1 { v0.b }[12], [x12] +; CHECK-SD-NEXT: ld1 { v6.b }[3], [x13] +; CHECK-SD-NEXT: add x13, sp, #864 +; CHECK-SD-NEXT: ld1 { v7.b }[2], [x14] +; CHECK-SD-NEXT: add x14, sp, #1016 +; CHECK-SD-NEXT: ld1 { v5.b }[2], [x13] +; CHECK-SD-NEXT: add x13, sp, #752 +; CHECK-SD-NEXT: ld1 { v4.b }[5], [x14] +; CHECK-SD-NEXT: add x14, sp, #104 ; CHECK-SD-NEXT: ld1 { v6.b }[4], [x13] -; CHECK-SD-NEXT: add x12, sp, #888 -; CHECK-SD-NEXT: ld1 { v1.b }[9], [x10] -; CHECK-SD-NEXT: ld1 { v18.b }[6], [x11] -; CHECK-SD-NEXT: add x11, sp, #1272 -; CHECK-SD-NEXT: add x10, sp, #776 -; CHECK-SD-NEXT: ld1 { v5.b }[7], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[5], [x11] -; CHECK-SD-NEXT: add x9, sp, #1168 -; CHECK-SD-NEXT: ld1 { v6.b }[5], [x12] -; CHECK-SD-NEXT: add x12, sp, #1280 -; CHECK-SD-NEXT: add x13, sp, #896 -; CHECK-SD-NEXT: ld1 { v18.b }[7], [x10] -; CHECK-SD-NEXT: add x11, sp, #784 -; CHECK-SD-NEXT: add x10, sp, #32 -; CHECK-SD-NEXT: ld1 { v5.b }[8], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[6], [x12] -; CHECK-SD-NEXT: add x9, sp, #1176 -; CHECK-SD-NEXT: ld1 { v6.b }[6], [x13] -; CHECK-SD-NEXT: add x12, sp, #904 -; CHECK-SD-NEXT: ld1 { v1.b }[10], [x10] -; CHECK-SD-NEXT: ld1 { v18.b }[8], [x11] -; CHECK-SD-NEXT: add x11, sp, #1288 -; CHECK-SD-NEXT: add x10, sp, #792 -; CHECK-SD-NEXT: ld1 { v5.b }[9], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[7], [x11] -; CHECK-SD-NEXT: add x9, sp, #1184 -; CHECK-SD-NEXT: ld1 { v6.b }[7], [x12] -; CHECK-SD-NEXT: add x12, sp, #1296 +; CHECK-SD-NEXT: add x13, sp, #1024 +; CHECK-SD-NEXT: ld1 { v7.b }[3], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[3], [x15] +; CHECK-SD-NEXT: add x15, sp, #760 +; CHECK-SD-NEXT: add x14, sp, #112 +; CHECK-SD-NEXT: ld1 { v4.b }[6], [x13] +; CHECK-SD-NEXT: add x13, sp, #880 +; CHECK-SD-NEXT: ld1 { v6.b }[5], [x15] +; CHECK-SD-NEXT: add x15, sp, #1032 +; CHECK-SD-NEXT: ld1 { v7.b }[4], [x14] +; CHECK-SD-NEXT: ld1 { v5.b }[4], [x13] +; CHECK-SD-NEXT: add x14, sp, #768 +; CHECK-SD-NEXT: add x13, sp, #120 +; CHECK-SD-NEXT: ld1 { v4.b }[7], [x15] +; CHECK-SD-NEXT: add x15, sp, #1040 +; CHECK-SD-NEXT: ld1 { v6.b }[6], [x14] +; CHECK-SD-NEXT: ld1 { v7.b }[5], [x13] +; CHECK-SD-NEXT: add x13, sp, #776 +; CHECK-SD-NEXT: ld1 { v5.b }[5], [x16] +; CHECK-SD-NEXT: add x14, sp, #1048 +; CHECK-SD-NEXT: ld1 { v4.b }[8], [x15] +; CHECK-SD-NEXT: add x15, sp, #896 +; CHECK-SD-NEXT: ld1 { v6.b }[7], [x13] +; CHECK-SD-NEXT: ld1 { v7.b }[6], [x10] +; CHECK-SD-NEXT: add x10, sp, #784 +; CHECK-SD-NEXT: ld1 { v5.b }[6], [x15] +; CHECK-SD-NEXT: add x13, sp, #1056 +; CHECK-SD-NEXT: ld1 { v4.b }[9], [x14] +; CHECK-SD-NEXT: add x14, sp, #904 +; CHECK-SD-NEXT: ld1 { v6.b }[8], [x10] +; CHECK-SD-NEXT: ld1 { v7.b }[7], [x9] +; CHECK-SD-NEXT: add x9, sp, #792 +; CHECK-SD-NEXT: ld1 { v5.b }[7], [x14] +; CHECK-SD-NEXT: add x10, sp, #1064 +; CHECK-SD-NEXT: ld1 { v4.b }[10], [x13] ; CHECK-SD-NEXT: add x13, sp, #912 -; CHECK-SD-NEXT: ld1 { v18.b }[9], [x10] -; CHECK-SD-NEXT: add x11, sp, #800 -; CHECK-SD-NEXT: add x10, sp, #40 +; CHECK-SD-NEXT: ld1 { v6.b }[9], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[8], [x8] +; CHECK-SD-NEXT: add x9, sp, #800 +; CHECK-SD-NEXT: ld1 { v5.b }[8], [x13] +; CHECK-SD-NEXT: add x8, sp, #152 +; CHECK-SD-NEXT: ld1 { v4.b }[11], [x10] +; CHECK-SD-NEXT: add x10, sp, #1072 +; CHECK-SD-NEXT: ld1 { v6.b }[10], [x9] +; CHECK-SD-NEXT: ld1 { v7.b }[9], [x8] +; CHECK-SD-NEXT: add x9, sp, #808 +; CHECK-SD-NEXT: ld1 { v5.b }[9], [x11] +; CHECK-SD-NEXT: add x8, sp, #56 +; CHECK-SD-NEXT: ld1 { v4.b }[12], [x10] +; CHECK-SD-NEXT: add x10, sp, #160 +; CHECK-SD-NEXT: ld1 { v0.b }[13], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[11], [x9] +; CHECK-SD-NEXT: add x9, sp, #928 +; CHECK-SD-NEXT: ld1 { v7.b }[10], [x10] +; CHECK-SD-NEXT: add x10, sp, #1080 ; CHECK-SD-NEXT: ld1 { v5.b }[10], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[8], [x12] -; CHECK-SD-NEXT: add x9, sp, #1192 -; CHECK-SD-NEXT: ld1 { v6.b }[8], [x13] -; CHECK-SD-NEXT: add x12, sp, #920 -; CHECK-SD-NEXT: ld1 { v1.b }[11], [x10] -; CHECK-SD-NEXT: ld1 { v18.b }[10], [x11] -; CHECK-SD-NEXT: add x11, sp, #1304 -; CHECK-SD-NEXT: add x10, sp, #808 -; CHECK-SD-NEXT: ld1 { v5.b }[11], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[9], [x11] -; CHECK-SD-NEXT: add x9, sp, #1200 -; CHECK-SD-NEXT: ld1 { v6.b }[9], [x12] -; CHECK-SD-NEXT: add x12, sp, #1312 -; CHECK-SD-NEXT: add x13, sp, #928 -; CHECK-SD-NEXT: ld1 { v18.b }[11], [x10] -; CHECK-SD-NEXT: add x11, sp, #816 -; CHECK-SD-NEXT: add x10, sp, #48 -; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[10], [x12] -; CHECK-SD-NEXT: add x9, sp, #1208 -; CHECK-SD-NEXT: ld1 { v6.b }[10], [x13] -; CHECK-SD-NEXT: add x12, sp, #936 -; CHECK-SD-NEXT: ld1 { v1.b }[12], [x10] -; CHECK-SD-NEXT: ld1 { v18.b }[12], [x11] -; CHECK-SD-NEXT: add x11, sp, #1320 -; CHECK-SD-NEXT: add x10, sp, #824 -; CHECK-SD-NEXT: ld1 { v5.b }[13], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[11], [x11] -; CHECK-SD-NEXT: add x9, sp, #1216 -; CHECK-SD-NEXT: ld1 { v6.b }[11], [x12] -; CHECK-SD-NEXT: add x12, sp, #1328 -; CHECK-SD-NEXT: add x13, sp, #944 -; CHECK-SD-NEXT: ld1 { v18.b }[13], [x10] -; CHECK-SD-NEXT: add x11, sp, #832 -; CHECK-SD-NEXT: add x10, sp, #56 -; CHECK-SD-NEXT: ld1 { v5.b }[14], [x9] -; CHECK-SD-NEXT: ld1 { v4.b }[12], [x12] -; CHECK-SD-NEXT: add x9, sp, #1224 -; CHECK-SD-NEXT: ld1 { v6.b }[12], [x13] -; CHECK-SD-NEXT: ld1 { v1.b }[13], [x10] -; CHECK-SD-NEXT: add x10, sp, #1336 -; CHECK-SD-NEXT: ld1 { v18.b }[14], [x11] -; CHECK-SD-NEXT: add x11, sp, #952 -; CHECK-SD-NEXT: add x8, sp, #448 -; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #840 +; CHECK-SD-NEXT: add x8, sp, #816 ; CHECK-SD-NEXT: ld1 { v4.b }[13], [x10] -; CHECK-SD-NEXT: ld1 { v6.b }[13], [x11] -; CHECK-SD-NEXT: usdot v17.4s, v16.16b, v7.16b -; CHECK-SD-NEXT: movi v7.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v18.b }[15], [x9] -; CHECK-SD-NEXT: add x9, sp, #64 -; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] -; CHECK-SD-NEXT: ld1 { v1.b }[14], [x9] -; CHECK-SD-NEXT: add x9, sp, #1344 -; CHECK-SD-NEXT: add x10, sp, #960 +; CHECK-SD-NEXT: add x9, sp, #168 +; CHECK-SD-NEXT: add x10, sp, #176 +; CHECK-SD-NEXT: ld1 { v6.b }[12], [x8] +; CHECK-SD-NEXT: add x8, sp, #936 +; CHECK-SD-NEXT: ld1 { v7.b }[11], [x9] +; CHECK-SD-NEXT: add x9, sp, #1088 +; CHECK-SD-NEXT: ld1 { v5.b }[11], [x8] +; CHECK-SD-NEXT: add x8, sp, #64 ; CHECK-SD-NEXT: ld1 { v4.b }[14], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[14], [x10] -; CHECK-SD-NEXT: add x8, sp, #456 -; CHECK-SD-NEXT: usdot v7.4s, v18.16b, v5.16b -; CHECK-SD-NEXT: movi v5.2d, #0000000000000000 -; CHECK-SD-NEXT: ld1 { v0.b }[15], [x8] -; CHECK-SD-NEXT: add x8, sp, #72 -; CHECK-SD-NEXT: movi v16.2d, #0000000000000000 -; CHECK-SD-NEXT: add x9, sp, #1352 -; CHECK-SD-NEXT: add x10, sp, #968 -; CHECK-SD-NEXT: ld1 { v1.b }[15], [x8] -; CHECK-SD-NEXT: ld1 { v4.b }[15], [x9] -; CHECK-SD-NEXT: ld1 { v6.b }[15], [x10] -; CHECK-SD-NEXT: usdot v16.4s, v1.16b, v0.16b +; CHECK-SD-NEXT: add x9, sp, #824 +; CHECK-SD-NEXT: ld1 { v0.b }[14], [x8] +; CHECK-SD-NEXT: ld1 { v6.b }[13], [x9] +; CHECK-SD-NEXT: add x9, sp, #944 +; CHECK-SD-NEXT: ld1 { v7.b }[12], [x10] +; CHECK-SD-NEXT: add x10, sp, #1096 +; CHECK-SD-NEXT: ld1 { v5.b }[12], [x9] +; CHECK-SD-NEXT: add x8, sp, #832 +; CHECK-SD-NEXT: ld1 { v4.b }[15], [x10] +; CHECK-SD-NEXT: add x9, sp, #184 +; CHECK-SD-NEXT: add x10, sp, #72 +; CHECK-SD-NEXT: ld1 { v6.b }[14], [x8] +; CHECK-SD-NEXT: add x8, sp, #952 +; CHECK-SD-NEXT: ld1 { v7.b }[13], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[13], [x8] +; CHECK-SD-NEXT: add x8, sp, #840 +; CHECK-SD-NEXT: ld1 { v0.b }[15], [x10] +; CHECK-SD-NEXT: sdot v2.4s, v4.16b, v1.16b +; CHECK-SD-NEXT: add x9, sp, #192 +; CHECK-SD-NEXT: ld1 { v6.b }[15], [x8] +; CHECK-SD-NEXT: add x8, sp, #960 +; CHECK-SD-NEXT: ld1 { v7.b }[14], [x9] +; CHECK-SD-NEXT: ld1 { v5.b }[14], [x8] +; CHECK-SD-NEXT: sdot v3.4s, v0.16b, v1.16b +; CHECK-SD-NEXT: add x8, sp, #200 +; CHECK-SD-NEXT: add x9, sp, #968 +; CHECK-SD-NEXT: sdot v2.4s, v6.16b, v1.16b +; CHECK-SD-NEXT: ld1 { v7.b }[15], [x8] +; CHECK-SD-NEXT: ld1 { v5.b }[15], [x9] +; CHECK-SD-NEXT: sdot v3.4s, v7.16b, v1.16b +; CHECK-SD-NEXT: sdot v2.4s, v5.16b, v1.16b ; CHECK-SD-NEXT: add v0.4s, v3.4s, v2.4s -; CHECK-SD-NEXT: add v1.4s, v7.4s, v17.4s -; CHECK-SD-NEXT: usdot v5.4s, v6.16b, v4.16b -; CHECK-SD-NEXT: add v0.4s, v16.4s, v0.4s -; CHECK-SD-NEXT: add v1.4s, v1.4s, v5.4s -; CHECK-SD-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-SD-NEXT: addv s0, v0.4s ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-SD-NEXT: ret ; -; CHECK-GI-LABEL: test_usdot_v48i8_double: +; CHECK-GI-LABEL: test_sdot_v48i8_double_nomla: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 ; CHECK-GI-NEXT: .cfi_offset w29, -16 -; CHECK-GI-NEXT: ldr w8, [sp, #16] -; CHECK-GI-NEXT: ldr w9, [sp, #80] -; CHECK-GI-NEXT: fmov s2, w0 -; CHECK-GI-NEXT: ldr w10, [sp, #24] -; CHECK-GI-NEXT: ldr w11, [sp, #144] -; CHECK-GI-NEXT: ldr w12, [sp, #296] -; CHECK-GI-NEXT: fmov s3, w8 +; CHECK-GI-NEXT: ldr w10, [sp, #80] +; CHECK-GI-NEXT: ldr w11, [sp, #208] +; CHECK-GI-NEXT: fmov s0, w0 +; CHECK-GI-NEXT: ldr w9, [sp, #88] +; CHECK-GI-NEXT: ldr w12, [sp, #728] +; CHECK-GI-NEXT: movi v6.16b, #1 +; CHECK-GI-NEXT: fmov s1, w10 +; CHECK-GI-NEXT: fmov s2, w11 +; CHECK-GI-NEXT: ldr w11, [sp, #720] +; CHECK-GI-NEXT: ldr w10, [sp, #216] +; CHECK-GI-NEXT: mov v0.b[1], w1 +; CHECK-GI-NEXT: ldr w13, [sp, #856] +; CHECK-GI-NEXT: fmov s3, w11 +; CHECK-GI-NEXT: ldr w8, [sp, #96] +; CHECK-GI-NEXT: ldr w11, [sp, #224] +; CHECK-GI-NEXT: mov v1.b[1], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #848] +; CHECK-GI-NEXT: mov v2.b[1], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #976] +; CHECK-GI-NEXT: movi v7.2d, #0000000000000000 +; CHECK-GI-NEXT: movi v16.2d, #0000000000000000 ; CHECK-GI-NEXT: fmov s4, w9 -; CHECK-GI-NEXT: ldr w8, [sp, #88] -; CHECK-GI-NEXT: fmov s5, w11 -; CHECK-GI-NEXT: ldr w9, [sp, #152] -; CHECK-GI-NEXT: ldr w11, [sp, #216] -; CHECK-GI-NEXT: ldr w13, [sp, #64] -; CHECK-GI-NEXT: ldr w14, [sp, #128] -; CHECK-GI-NEXT: mov v2.b[1], w1 -; CHECK-GI-NEXT: mov v3.b[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #208] -; CHECK-GI-NEXT: mov v4.b[1], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #32] +; CHECK-GI-NEXT: mov v3.b[1], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #984] +; CHECK-GI-NEXT: fmov s5, w10 +; CHECK-GI-NEXT: mov v0.b[2], w2 +; CHECK-GI-NEXT: ldr w10, [sp, #736] +; CHECK-GI-NEXT: mov v1.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #864] +; CHECK-GI-NEXT: mov v2.b[2], w11 +; CHECK-GI-NEXT: mov v4.b[1], w13 +; CHECK-GI-NEXT: ldr w11, [sp, #992] +; CHECK-GI-NEXT: ldr w12, [sp, #776] ; CHECK-GI-NEXT: mov v5.b[1], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #96] -; CHECK-GI-NEXT: fmov s6, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #224] -; CHECK-GI-NEXT: ldr w15, [sp, #264] -; CHECK-GI-NEXT: ldr w16, [sp, #528] -; CHECK-GI-NEXT: mov v2.b[2], w2 -; CHECK-GI-NEXT: ldr w17, [sp, #648] -; CHECK-GI-NEXT: mov v3.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #160] -; CHECK-GI-NEXT: mov v4.b[2], w9 -; CHECK-GI-NEXT: mov v6.b[1], w11 -; CHECK-GI-NEXT: ldr w9, [sp, #272] -; CHECK-GI-NEXT: ldr w11, [sp, #48] -; CHECK-GI-NEXT: mov v5.b[2], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #40] -; CHECK-GI-NEXT: fmov s18, w16 -; CHECK-GI-NEXT: fmov s0, w9 -; CHECK-GI-NEXT: ldr w9, [sp, #168] -; CHECK-GI-NEXT: mov v2.b[3], w3 +; CHECK-GI-NEXT: mov v3.b[2], w10 +; CHECK-GI-NEXT: ldr w9, [sp, #104] +; CHECK-GI-NEXT: ldr w10, [sp, #232] +; CHECK-GI-NEXT: mov v0.b[3], w3 +; CHECK-GI-NEXT: movi v17.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v1.b[3], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #872] +; CHECK-GI-NEXT: movi v18.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v4.b[2], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #744] +; CHECK-GI-NEXT: mov v2.b[3], w10 +; CHECK-GI-NEXT: mov v5.b[2], w11 +; CHECK-GI-NEXT: ldr w11, [sp, #1000] +; CHECK-GI-NEXT: ldr w10, [sp, #240] ; CHECK-GI-NEXT: mov v3.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #232] -; CHECK-GI-NEXT: ldr w16, [sp, #584] -; CHECK-GI-NEXT: mov v6.b[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #104] -; CHECK-GI-NEXT: mov v5.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #56] -; CHECK-GI-NEXT: mov v4.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #280] -; CHECK-GI-NEXT: mov v2.b[4], w4 -; CHECK-GI-NEXT: mov v3.b[4], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #240] -; CHECK-GI-NEXT: mov v6.b[3], w8 ; CHECK-GI-NEXT: ldr w8, [sp, #112] -; CHECK-GI-NEXT: mov v0.b[1], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #176] -; CHECK-GI-NEXT: mov v4.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #288] -; CHECK-GI-NEXT: mov v2.b[5], w5 -; CHECK-GI-NEXT: mov v5.b[4], w10 -; CHECK-GI-NEXT: mov v3.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #248] -; CHECK-GI-NEXT: mov v0.b[2], w8 -; CHECK-GI-NEXT: mov v6.b[4], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #120] -; CHECK-GI-NEXT: ldr w8, [sp, #184] -; CHECK-GI-NEXT: ldr w10, [sp, #72] -; CHECK-GI-NEXT: mov v4.b[5], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #136] -; CHECK-GI-NEXT: mov v2.b[6], w6 -; CHECK-GI-NEXT: mov v5.b[5], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #192] -; CHECK-GI-NEXT: mov v3.b[6], w13 -; CHECK-GI-NEXT: mov v0.b[3], w12 -; CHECK-GI-NEXT: mov v6.b[5], w9 -; CHECK-GI-NEXT: ldr w13, [sp, #304] -; CHECK-GI-NEXT: ldr w9, [sp, #256] -; CHECK-GI-NEXT: ldr w12, [sp, #200] -; CHECK-GI-NEXT: mov v4.b[6], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #336] -; CHECK-GI-NEXT: mov v2.b[7], w7 -; CHECK-GI-NEXT: mov v5.b[6], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #344] -; CHECK-GI-NEXT: mov v3.b[7], w10 -; CHECK-GI-NEXT: mov v0.b[4], w13 -; CHECK-GI-NEXT: fmov s7, w14 -; CHECK-GI-NEXT: mov v6.b[6], w9 -; CHECK-GI-NEXT: ldr w14, [sp, #312] -; CHECK-GI-NEXT: ldr w13, [sp, #352] -; CHECK-GI-NEXT: ldr w10, [sp, #360] -; CHECK-GI-NEXT: mov v4.b[7], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #472] -; CHECK-GI-NEXT: ldr w9, [sp, #320] -; CHECK-GI-NEXT: mov v5.b[7], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #400] -; CHECK-GI-NEXT: mov v7.b[1], w8 -; CHECK-GI-NEXT: mov v0.b[5], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #464] -; CHECK-GI-NEXT: mov v6.b[7], w15 -; CHECK-GI-NEXT: fmov s16, w12 -; CHECK-GI-NEXT: ldr w15, [sp, #408] -; CHECK-GI-NEXT: ldr w8, [sp, #328] -; CHECK-GI-NEXT: fmov s17, w14 -; CHECK-GI-NEXT: ldr w14, [sp, #592] -; CHECK-GI-NEXT: ldr w12, [sp, #368] -; CHECK-GI-NEXT: mov v7.b[2], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #536] -; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 -; CHECK-GI-NEXT: mov v16.b[1], w15 -; CHECK-GI-NEXT: fmov s19, w14 -; CHECK-GI-NEXT: ldr w15, [sp, #600] -; CHECK-GI-NEXT: mov v18.b[1], w13 -; CHECK-GI-NEXT: ldr w14, [sp, #416] -; CHECK-GI-NEXT: mov v17.b[1], w11 -; CHECK-GI-NEXT: mov v0.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #480] -; CHECK-GI-NEXT: ldr w11, [sp, #376] -; CHECK-GI-NEXT: mov v19.b[1], w15 -; CHECK-GI-NEXT: mov v7.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #544] -; CHECK-GI-NEXT: mov v16.b[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #608] -; CHECK-GI-NEXT: ldr w13, [sp, #384] -; CHECK-GI-NEXT: mov v18.b[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #424] -; CHECK-GI-NEXT: mov v17.b[2], w9 -; CHECK-GI-NEXT: mov v0.b[7], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #488] -; CHECK-GI-NEXT: ldr w9, [sp, #392] -; CHECK-GI-NEXT: mov v19.b[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #552] -; CHECK-GI-NEXT: mov v7.b[4], w12 -; CHECK-GI-NEXT: mov v16.b[3], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #656] -; CHECK-GI-NEXT: ldr w12, [sp, #616] -; CHECK-GI-NEXT: mov v17.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #432] -; CHECK-GI-NEXT: mov v18.b[3], w14 -; CHECK-GI-NEXT: fmov s1, w10 -; CHECK-GI-NEXT: ldr w14, [sp, #664] -; CHECK-GI-NEXT: ldr w10, [sp, #440] -; CHECK-GI-NEXT: mov v19.b[3], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #496] -; CHECK-GI-NEXT: mov v7.b[5], w11 -; CHECK-GI-NEXT: mov v16.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #624] -; CHECK-GI-NEXT: ldr w11, [sp, #504] -; CHECK-GI-NEXT: mov v1.b[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #560] -; CHECK-GI-NEXT: mov v17.b[4], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #448] -; CHECK-GI-NEXT: ldr w15, [sp, #456] -; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-GI-NEXT: mov v18.b[4], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #672] -; CHECK-GI-NEXT: mov v19.b[4], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #568] -; CHECK-GI-NEXT: mov v16.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #632] -; CHECK-GI-NEXT: mov v1.b[2], w14 -; CHECK-GI-NEXT: mov v17.b[5], w11 -; CHECK-GI-NEXT: mov v7.b[6], w13 -; CHECK-GI-NEXT: ldr w11, [sp, #512] -; CHECK-GI-NEXT: ldr w13, [sp, #520] -; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 -; CHECK-GI-NEXT: mov v18.b[5], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #680] -; CHECK-GI-NEXT: mov v19.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #576] -; CHECK-GI-NEXT: mov v16.b[6], w12 -; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0 -; CHECK-GI-NEXT: mov v1.b[3], w8 -; CHECK-GI-NEXT: ldr w8, [sp, #640] -; CHECK-GI-NEXT: mov v17.b[6], w11 -; CHECK-GI-NEXT: mov v7.b[7], w9 -; CHECK-GI-NEXT: ushll v20.4s, v2.4h, #0 -; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-NEXT: mov v18.b[6], w10 -; CHECK-GI-NEXT: mov v19.b[6], w8 -; CHECK-GI-NEXT: ldr w10, [sp, #688] -; CHECK-GI-NEXT: mov v16.b[7], w15 -; CHECK-GI-NEXT: ushll v21.4s, v3.4h, #0 -; CHECK-GI-NEXT: ushll2 v22.4s, v3.8h, #0 -; CHECK-GI-NEXT: mov v17.b[7], w13 -; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0 -; CHECK-GI-NEXT: ushll2 v24.4s, v4.8h, #0 -; CHECK-GI-NEXT: sshll v7.8h, v7.8b, #0 -; CHECK-GI-NEXT: ushll v3.4s, v6.4h, #0 -; CHECK-GI-NEXT: ushll2 v4.4s, v6.8h, #0 -; CHECK-GI-NEXT: mov v18.b[7], w16 -; CHECK-GI-NEXT: mov v19.b[7], w17 -; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 -; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0 -; CHECK-GI-NEXT: mov v1.b[4], w10 -; CHECK-GI-NEXT: ldr w9, [sp, #696] -; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-GI-NEXT: ldr w14, [sp, #720] -; CHECK-GI-NEXT: ldr w13, [sp, #728] -; CHECK-GI-NEXT: ushll v25.4s, v5.4h, #0 -; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 -; CHECK-GI-NEXT: ldr w12, [sp, #704] -; CHECK-GI-NEXT: sshll v6.8h, v18.8b, #0 -; CHECK-GI-NEXT: sshll v18.8h, v19.8b, #0 -; CHECK-GI-NEXT: sshll v19.4s, v7.4h, #0 -; CHECK-GI-NEXT: sshll2 v7.4s, v7.8h, #0 -; CHECK-GI-NEXT: sshll v26.4s, v16.4h, #0 -; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0 -; CHECK-GI-NEXT: sshll v27.4s, v17.4h, #0 -; CHECK-GI-NEXT: mov v1.b[5], w9 -; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0 -; CHECK-GI-NEXT: mul v19.4s, v20.4s, v19.4s -; CHECK-GI-NEXT: sshll v28.4s, v6.4h, #0 -; CHECK-GI-NEXT: sshll2 v6.4s, v6.8h, #0 -; CHECK-GI-NEXT: mul v7.4s, v2.4s, v7.4s -; CHECK-GI-NEXT: mul v20.4s, v21.4s, v26.4s -; CHECK-GI-NEXT: mul v16.4s, v22.4s, v16.4s -; CHECK-GI-NEXT: mul v21.4s, v23.4s, v27.4s -; CHECK-GI-NEXT: fmov s2, w14 -; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0 -; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0 -; CHECK-GI-NEXT: mul v5.4s, v5.4s, v6.4s -; CHECK-GI-NEXT: mul v17.4s, v24.4s, v17.4s -; CHECK-GI-NEXT: addv s6, v19.4s -; CHECK-GI-NEXT: mov v1.b[6], w12 -; CHECK-GI-NEXT: ldr w11, [sp, #736] -; CHECK-GI-NEXT: addv s7, v7.4s -; CHECK-GI-NEXT: mov v2.b[1], w13 -; CHECK-GI-NEXT: addv s16, v16.4s -; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s -; CHECK-GI-NEXT: addv s18, v20.4s -; CHECK-GI-NEXT: addv s19, v21.4s -; CHECK-GI-NEXT: fmov w12, s6 -; CHECK-GI-NEXT: mul v22.4s, v25.4s, v28.4s -; CHECK-GI-NEXT: addv s17, v17.4s -; CHECK-GI-NEXT: fmov w13, s7 -; CHECK-GI-NEXT: mul v3.4s, v3.4s, v29.4s -; CHECK-GI-NEXT: addv s5, v5.4s -; CHECK-GI-NEXT: mov v2.b[2], w11 -; CHECK-GI-NEXT: fmov w14, s19 -; CHECK-GI-NEXT: ldr w10, [sp, #744] +; CHECK-GI-NEXT: mov v0.b[4], w4 +; CHECK-GI-NEXT: movi v19.2d, #0000000000000000 +; CHECK-GI-NEXT: movi v20.2d, #0000000000000000 +; CHECK-GI-NEXT: mov v4.b[3], w9 ; CHECK-GI-NEXT: ldr w9, [sp, #752] -; CHECK-GI-NEXT: ldr w8, [sp, #712] -; CHECK-GI-NEXT: ldr w15, [sp, #864] -; CHECK-GI-NEXT: add w11, w12, w13 -; CHECK-GI-NEXT: fmov w12, s18 -; CHECK-GI-NEXT: fmov w13, s16 -; CHECK-GI-NEXT: addv s20, v22.4s -; CHECK-GI-NEXT: addv s3, v3.4s -; CHECK-GI-NEXT: mov v1.b[7], w8 -; CHECK-GI-NEXT: mov v2.b[3], w10 -; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0 -; CHECK-GI-NEXT: add w11, w11, w12 -; CHECK-GI-NEXT: add w12, w13, w14 -; CHECK-GI-NEXT: fmov w13, s17 -; CHECK-GI-NEXT: fmov w10, s20 -; CHECK-GI-NEXT: ldr w14, [sp, #784] -; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #0 -; CHECK-GI-NEXT: ushll v22.4s, v0.4h, #0 -; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0 -; CHECK-GI-NEXT: add w12, w12, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #792] -; CHECK-GI-NEXT: mov v2.b[4], w9 -; CHECK-GI-NEXT: add w11, w11, w12 -; CHECK-GI-NEXT: fmov w12, s5 -; CHECK-GI-NEXT: addv s5, v4.4s -; CHECK-GI-NEXT: ldr w9, [sp, #856] -; CHECK-GI-NEXT: sshll v23.4s, v1.4h, #0 -; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0 -; CHECK-GI-NEXT: add w10, w10, w12 -; CHECK-GI-NEXT: fmov w12, s3 -; CHECK-GI-NEXT: fmov s3, w14 -; CHECK-GI-NEXT: ldr w14, [sp, #760] -; CHECK-GI-NEXT: mul v22.4s, v22.4s, v23.4s -; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s -; CHECK-GI-NEXT: add w12, w10, w12 -; CHECK-GI-NEXT: ldr w10, [sp, #848] -; CHECK-GI-NEXT: mov v3.b[1], w13 -; CHECK-GI-NEXT: fmov w13, s5 -; CHECK-GI-NEXT: mov v2.b[5], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #920] -; CHECK-GI-NEXT: fmov s4, w10 -; CHECK-GI-NEXT: ldr w10, [sp, #768] -; CHECK-GI-NEXT: addv s1, v22.4s -; CHECK-GI-NEXT: addv s0, v0.4s -; CHECK-GI-NEXT: add w12, w12, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #800] -; CHECK-GI-NEXT: mov v4.b[1], w9 -; CHECK-GI-NEXT: add w8, w11, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #912] -; CHECK-GI-NEXT: mov v3.b[2], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #976] -; CHECK-GI-NEXT: ldr w11, [sp, #984] -; CHECK-GI-NEXT: fmov s5, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1048] -; CHECK-GI-NEXT: mov v2.b[6], w10 -; CHECK-GI-NEXT: fmov s6, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #808] -; CHECK-GI-NEXT: ldr w10, [sp, #832] -; CHECK-GI-NEXT: mov v4.b[2], w15 -; CHECK-GI-NEXT: ldr w15, [sp, #1040] -; CHECK-GI-NEXT: ldr w9, [sp, #776] -; CHECK-GI-NEXT: mov v5.b[1], w14 -; CHECK-GI-NEXT: mov v3.b[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1056] -; CHECK-GI-NEXT: fmov s7, w15 -; CHECK-GI-NEXT: mov v6.b[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #872] -; CHECK-GI-NEXT: ldr w14, [sp, #816] -; CHECK-GI-NEXT: ldr w15, [sp, #992] -; CHECK-GI-NEXT: mov v2.b[7], w9 -; CHECK-GI-NEXT: mov v4.b[3], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #824] -; CHECK-GI-NEXT: ldr w9, [sp, #840] -; CHECK-GI-NEXT: mov v7.b[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #928] -; CHECK-GI-NEXT: mov v3.b[4], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1064] -; CHECK-GI-NEXT: mov v6.b[2], w15 -; CHECK-GI-NEXT: ldr w15, [sp, #1000] -; CHECK-GI-NEXT: mov v5.b[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #880] -; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0 -; CHECK-GI-NEXT: mov v7.b[2], w13 -; CHECK-GI-NEXT: mov v4.b[4], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1104] -; CHECK-GI-NEXT: ldr w13, [sp, #936] -; CHECK-GI-NEXT: mov v3.b[5], w11 -; CHECK-GI-NEXT: mov v6.b[3], w15 -; CHECK-GI-NEXT: fmov s16, w12 -; CHECK-GI-NEXT: ldr w12, [sp, #944] +; CHECK-GI-NEXT: mov v1.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #880] +; CHECK-GI-NEXT: mov v5.b[3], w11 +; CHECK-GI-NEXT: mov v2.b[4], w10 +; CHECK-GI-NEXT: mov v3.b[4], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #120] ; CHECK-GI-NEXT: ldr w11, [sp, #1008] -; CHECK-GI-NEXT: mov v5.b[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1112] -; CHECK-GI-NEXT: ldr w15, [sp, #904] -; CHECK-GI-NEXT: mov v7.b[3], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #888] -; CHECK-GI-NEXT: mov v16.b[1], w13 -; CHECK-GI-NEXT: mov v3.b[6], w10 -; CHECK-GI-NEXT: ldr w13, [sp, #896] -; CHECK-GI-NEXT: mov v4.b[5], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1072] -; CHECK-GI-NEXT: mov v6.b[4], w11 -; CHECK-GI-NEXT: mov v5.b[4], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1120] -; CHECK-GI-NEXT: ldr w11, [sp, #952] -; CHECK-GI-NEXT: mov v7.b[4], w14 -; CHECK-GI-NEXT: ldr w10, [sp, #1016] -; CHECK-GI-NEXT: ldr w14, [sp, #960] -; CHECK-GI-NEXT: mov v16.b[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1080] -; CHECK-GI-NEXT: mov v3.b[7], w9 -; CHECK-GI-NEXT: mov v4.b[6], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1128] -; CHECK-GI-NEXT: ldr w9, [sp, #1088] +; CHECK-GI-NEXT: ldr w10, [sp, #248] +; CHECK-GI-NEXT: mov v0.b[5], w5 +; CHECK-GI-NEXT: mov v4.b[4], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #760] +; CHECK-GI-NEXT: mov v1.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #888] +; CHECK-GI-NEXT: mov v5.b[4], w11 +; CHECK-GI-NEXT: mov v2.b[5], w10 +; CHECK-GI-NEXT: mov v3.b[5], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #128] +; CHECK-GI-NEXT: ldr w11, [sp, #1016] +; CHECK-GI-NEXT: ldr w10, [sp, #256] +; CHECK-GI-NEXT: mov v0.b[6], w6 +; CHECK-GI-NEXT: mov v4.b[5], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #768] +; CHECK-GI-NEXT: mov v1.b[6], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #896] ; CHECK-GI-NEXT: mov v5.b[5], w11 -; CHECK-GI-NEXT: mov v6.b[5], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1024] -; CHECK-GI-NEXT: mov v7.b[5], w12 -; CHECK-GI-NEXT: ldr w11, [sp, #968] +; CHECK-GI-NEXT: mov v2.b[6], w10 +; CHECK-GI-NEXT: mov v3.b[6], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #136] +; CHECK-GI-NEXT: ldr w11, [sp, #1024] +; CHECK-GI-NEXT: ldr w10, [sp, #264] +; CHECK-GI-NEXT: mov v0.b[7], w7 +; CHECK-GI-NEXT: mov v4.b[6], w8 +; CHECK-GI-NEXT: mov v1.b[7], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #904] +; CHECK-GI-NEXT: mov v5.b[6], w11 +; CHECK-GI-NEXT: mov v2.b[7], w10 +; CHECK-GI-NEXT: ldr w8, [sp, #16] +; CHECK-GI-NEXT: mov v3.b[7], w12 +; CHECK-GI-NEXT: ldr w10, [sp, #144] ; CHECK-GI-NEXT: ldr w12, [sp, #1032] -; CHECK-GI-NEXT: mov v16.b[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1136] -; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0 -; CHECK-GI-NEXT: mov v4.b[7], w15 -; CHECK-GI-NEXT: ldr w15, [sp, #1144] -; CHECK-GI-NEXT: mov v5.b[6], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1168] -; CHECK-GI-NEXT: mov v6.b[6], w10 -; CHECK-GI-NEXT: mov v7.b[6], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1232] -; CHECK-GI-NEXT: ldr w10, [sp, #1176] -; CHECK-GI-NEXT: mov v16.b[4], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1240] -; CHECK-GI-NEXT: fmov s17, w14 -; CHECK-GI-NEXT: fmov s18, w9 -; CHECK-GI-NEXT: ldr w14, [sp, #1096] -; CHECK-GI-NEXT: ldr w9, [sp, #1160] -; CHECK-GI-NEXT: mov v5.b[7], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1184] -; CHECK-GI-NEXT: mov v6.b[7], w12 -; CHECK-GI-NEXT: mov v17.b[1], w10 -; CHECK-GI-NEXT: ldr w12, [sp, #1248] -; CHECK-GI-NEXT: mov v7.b[7], w14 -; CHECK-GI-NEXT: mov v18.b[1], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1296] -; CHECK-GI-NEXT: mov v16.b[5], w15 -; CHECK-GI-NEXT: ldr w15, [sp, #1360] -; CHECK-GI-NEXT: ldr w14, [sp, #1304] -; CHECK-GI-NEXT: ldr w10, [sp, #1152] -; CHECK-GI-NEXT: fmov s19, w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1424] -; CHECK-GI-NEXT: ushll v4.8h, v4.8b, #0 -; CHECK-GI-NEXT: mov v17.b[2], w11 -; CHECK-GI-NEXT: fmov s20, w15 -; CHECK-GI-NEXT: ldr w11, [sp, #1368] -; CHECK-GI-NEXT: fmov s21, w13 -; CHECK-GI-NEXT: mov v18.b[2], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1432] -; CHECK-GI-NEXT: mov v19.b[1], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1192] -; CHECK-GI-NEXT: ldr w13, [sp, #1200] -; CHECK-GI-NEXT: mov v20.b[1], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1312] -; CHECK-GI-NEXT: mov v16.b[6], w10 -; CHECK-GI-NEXT: mov v21.b[1], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1256] -; CHECK-GI-NEXT: mov v17.b[3], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1440] -; CHECK-GI-NEXT: ldr w10, [sp, #1376] -; CHECK-GI-NEXT: ushll v22.4s, v3.4h, #0 -; CHECK-GI-NEXT: mov v18.b[3], w12 -; CHECK-GI-NEXT: mov v19.b[2], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1208] -; CHECK-GI-NEXT: mov v20.b[2], w10 -; CHECK-GI-NEXT: ldr w10, [sp, #1320] -; CHECK-GI-NEXT: mov v16.b[7], w9 -; CHECK-GI-NEXT: mov v21.b[2], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1264] -; CHECK-GI-NEXT: mov v17.b[4], w13 -; CHECK-GI-NEXT: ldr w9, [sp, #1384] -; CHECK-GI-NEXT: ldr w13, [sp, #1448] -; CHECK-GI-NEXT: ldr w12, [sp, #1216] -; CHECK-GI-NEXT: mov v18.b[4], w14 -; CHECK-GI-NEXT: mov v19.b[3], w10 -; CHECK-GI-NEXT: ldr w14, [sp, #1272] -; CHECK-GI-NEXT: mov v20.b[3], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1328] -; CHECK-GI-NEXT: ldr w10, [sp, #1224] -; CHECK-GI-NEXT: mov v17.b[5], w11 -; CHECK-GI-NEXT: mov v21.b[3], w13 -; CHECK-GI-NEXT: ldr w13, [sp, #1280] -; CHECK-GI-NEXT: ldr w11, [sp, #1392] -; CHECK-GI-NEXT: sshll v16.8h, v16.8b, #0 -; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0 -; CHECK-GI-NEXT: mov v18.b[5], w14 -; CHECK-GI-NEXT: mov v19.b[4], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1336] -; CHECK-GI-NEXT: ldr w14, [sp, #1456] -; CHECK-GI-NEXT: mov v20.b[4], w11 -; CHECK-GI-NEXT: ldr w11, [sp, #1288] -; CHECK-GI-NEXT: mov v17.b[6], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1344] -; CHECK-GI-NEXT: sshll v27.4s, v16.4h, #0 -; CHECK-GI-NEXT: mov v21.b[4], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1400] -; CHECK-GI-NEXT: sshll2 v16.4s, v16.8h, #0 -; CHECK-GI-NEXT: mov v18.b[6], w13 -; CHECK-GI-NEXT: mov v19.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1464] -; CHECK-GI-NEXT: mov v20.b[5], w14 -; CHECK-GI-NEXT: ldr w14, [sp, #1352] -; CHECK-GI-NEXT: ldr w13, [sp, #1416] -; CHECK-GI-NEXT: mov v17.b[7], w10 -; CHECK-GI-NEXT: ushll v23.4s, v4.4h, #0 -; CHECK-GI-NEXT: ushll v5.8h, v5.8b, #0 -; CHECK-GI-NEXT: mov v21.b[5], w9 -; CHECK-GI-NEXT: ldr w9, [sp, #1408] -; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0 -; CHECK-GI-NEXT: mov v18.b[7], w11 -; CHECK-GI-NEXT: mov v19.b[6], w12 -; CHECK-GI-NEXT: ldr w12, [sp, #1472] -; CHECK-GI-NEXT: mov v20.b[6], w9 +; CHECK-GI-NEXT: mov v0.b[8], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #784] +; CHECK-GI-NEXT: ldr w11, [sp, #272] +; CHECK-GI-NEXT: mov v4.b[7], w9 +; CHECK-GI-NEXT: mov v1.b[8], w10 +; CHECK-GI-NEXT: ldr w10, [sp, #912] +; CHECK-GI-NEXT: mov v5.b[7], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #24] +; CHECK-GI-NEXT: ldr w12, [sp, #1040] +; CHECK-GI-NEXT: mov v3.b[8], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #152] +; CHECK-GI-NEXT: mov v2.b[8], w11 +; CHECK-GI-NEXT: mov v0.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #792] +; CHECK-GI-NEXT: ldr w11, [sp, #280] +; CHECK-GI-NEXT: mov v4.b[8], w10 +; CHECK-GI-NEXT: mov v1.b[9], w8 +; CHECK-GI-NEXT: ldr w10, [sp, #920] +; CHECK-GI-NEXT: mov v5.b[8], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #32] +; CHECK-GI-NEXT: ldr w12, [sp, #1048] +; CHECK-GI-NEXT: mov v3.b[9], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #160] +; CHECK-GI-NEXT: mov v2.b[9], w11 +; CHECK-GI-NEXT: mov v0.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #800] +; CHECK-GI-NEXT: ldr w11, [sp, #288] +; CHECK-GI-NEXT: mov v4.b[9], w10 +; CHECK-GI-NEXT: mov v1.b[10], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #928] +; CHECK-GI-NEXT: mov v5.b[9], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #40] +; CHECK-GI-NEXT: ldr w12, [sp, #1056] +; CHECK-GI-NEXT: mov v3.b[10], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #168] +; CHECK-GI-NEXT: mov v2.b[10], w11 +; CHECK-GI-NEXT: mov v0.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #808] +; CHECK-GI-NEXT: ldr w11, [sp, #296] +; CHECK-GI-NEXT: mov v4.b[10], w10 +; CHECK-GI-NEXT: mov v1.b[11], w8 +; CHECK-GI-NEXT: ldr w10, [sp, #936] +; CHECK-GI-NEXT: mov v5.b[10], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #48] +; CHECK-GI-NEXT: ldr w12, [sp, #1064] +; CHECK-GI-NEXT: mov v3.b[11], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #176] +; CHECK-GI-NEXT: mov v2.b[11], w11 +; CHECK-GI-NEXT: mov v0.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #816] +; CHECK-GI-NEXT: ldr w11, [sp, #304] +; CHECK-GI-NEXT: mov v4.b[11], w10 +; CHECK-GI-NEXT: mov v1.b[12], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #944] +; CHECK-GI-NEXT: mov v5.b[11], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #56] +; CHECK-GI-NEXT: ldr w12, [sp, #1072] +; CHECK-GI-NEXT: mov v3.b[12], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #184] +; CHECK-GI-NEXT: mov v2.b[12], w11 +; CHECK-GI-NEXT: mov v0.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #824] +; CHECK-GI-NEXT: ldr w11, [sp, #312] +; CHECK-GI-NEXT: mov v4.b[12], w10 +; CHECK-GI-NEXT: mov v1.b[13], w8 +; CHECK-GI-NEXT: ldr w10, [sp, #952] +; CHECK-GI-NEXT: mov v5.b[12], w12 +; CHECK-GI-NEXT: ldr w8, [sp, #64] +; CHECK-GI-NEXT: ldr w12, [sp, #1080] +; CHECK-GI-NEXT: mov v3.b[13], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #192] +; CHECK-GI-NEXT: mov v2.b[13], w11 +; CHECK-GI-NEXT: mov v0.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #832] +; CHECK-GI-NEXT: ldr w11, [sp, #320] +; CHECK-GI-NEXT: mov v4.b[13], w10 +; CHECK-GI-NEXT: mov v1.b[14], w9 +; CHECK-GI-NEXT: ldr w10, [sp, #960] +; CHECK-GI-NEXT: mov v5.b[13], w12 +; CHECK-GI-NEXT: ldr w9, [sp, #72] +; CHECK-GI-NEXT: ldr w12, [sp, #1088] +; CHECK-GI-NEXT: mov v3.b[14], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #200] +; CHECK-GI-NEXT: mov v2.b[14], w11 +; CHECK-GI-NEXT: mov v0.b[15], w9 +; CHECK-GI-NEXT: ldr w9, [sp, #840] +; CHECK-GI-NEXT: ldr w11, [sp, #328] +; CHECK-GI-NEXT: mov v4.b[14], w10 +; CHECK-GI-NEXT: mov v1.b[15], w8 +; CHECK-GI-NEXT: ldr w8, [sp, #968] +; CHECK-GI-NEXT: mov v5.b[14], w12 +; CHECK-GI-NEXT: ldr w10, [sp, #1096] +; CHECK-GI-NEXT: mov v3.b[15], w9 +; CHECK-GI-NEXT: mov v2.b[15], w11 +; CHECK-GI-NEXT: sdot v7.4s, v0.16b, v6.16b +; CHECK-GI-NEXT: mov v4.b[15], w8 +; CHECK-GI-NEXT: sdot v16.4s, v1.16b, v6.16b +; CHECK-GI-NEXT: mov v5.b[15], w10 +; CHECK-GI-NEXT: sdot v17.4s, v3.16b, v6.16b +; CHECK-GI-NEXT: sdot v20.4s, v2.16b, v6.16b +; CHECK-GI-NEXT: addv s0, v7.4s +; CHECK-GI-NEXT: sdot v18.4s, v4.16b, v6.16b +; CHECK-GI-NEXT: addv s1, v16.4s +; CHECK-GI-NEXT: sdot v19.4s, v5.16b, v6.16b +; CHECK-GI-NEXT: addv s2, v17.4s +; CHECK-GI-NEXT: addv s4, v20.4s +; CHECK-GI-NEXT: fmov w8, s0 ; CHECK-GI-NEXT: fmov w9, s1 -; CHECK-GI-NEXT: ushll v1.4s, v2.4h, #0 -; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0 -; CHECK-GI-NEXT: sshll v17.8h, v17.8b, #0 -; CHECK-GI-NEXT: ldr w15, [sp, #1480] -; CHECK-GI-NEXT: mov v21.b[6], w12 -; CHECK-GI-NEXT: ushll v24.4s, v5.4h, #0 -; CHECK-GI-NEXT: ushll v6.8h, v6.8b, #0 -; CHECK-GI-NEXT: sshll v18.8h, v18.8b, #0 -; CHECK-GI-NEXT: mov v19.b[7], w14 -; CHECK-GI-NEXT: mul v1.4s, v1.4s, v27.4s -; CHECK-GI-NEXT: sshll v28.4s, v17.4h, #0 -; CHECK-GI-NEXT: sshll2 v17.4s, v17.8h, #0 -; CHECK-GI-NEXT: mul v2.4s, v2.4s, v16.4s -; CHECK-GI-NEXT: mov v20.b[7], w13 -; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0 -; CHECK-GI-NEXT: ushll v25.4s, v6.4h, #0 -; CHECK-GI-NEXT: sshll v29.4s, v18.4h, #0 -; CHECK-GI-NEXT: sshll2 v18.4s, v18.8h, #0 -; CHECK-GI-NEXT: mov v21.b[7], w15 -; CHECK-GI-NEXT: sshll v19.8h, v19.8b, #0 -; CHECK-GI-NEXT: mul v16.4s, v22.4s, v28.4s -; CHECK-GI-NEXT: mul v3.4s, v3.4s, v17.4s -; CHECK-GI-NEXT: addv s1, v1.4s -; CHECK-GI-NEXT: addv s2, v2.4s -; CHECK-GI-NEXT: ushll v7.8h, v7.8b, #0 -; CHECK-GI-NEXT: mul v17.4s, v23.4s, v29.4s -; CHECK-GI-NEXT: sshll v20.8h, v20.8b, #0 -; CHECK-GI-NEXT: mul v4.4s, v4.4s, v18.4s -; CHECK-GI-NEXT: sshll v30.4s, v19.4h, #0 -; CHECK-GI-NEXT: sshll2 v19.4s, v19.8h, #0 -; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0 -; CHECK-GI-NEXT: addv s16, v16.4s -; CHECK-GI-NEXT: addv s3, v3.4s -; CHECK-GI-NEXT: fmov w10, s1 -; CHECK-GI-NEXT: fmov w11, s2 -; CHECK-GI-NEXT: sshll v31.4s, v20.4h, #0 -; CHECK-GI-NEXT: sshll v21.8h, v21.8b, #0 -; CHECK-GI-NEXT: addv s17, v17.4s -; CHECK-GI-NEXT: mul v18.4s, v24.4s, v30.4s -; CHECK-GI-NEXT: mul v5.4s, v5.4s, v19.4s -; CHECK-GI-NEXT: addv s4, v4.4s -; CHECK-GI-NEXT: fmov w12, s3 -; CHECK-GI-NEXT: sshll2 v20.4s, v20.8h, #0 -; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: fmov w11, s16 -; CHECK-GI-NEXT: mul v19.4s, v25.4s, v31.4s -; CHECK-GI-NEXT: fmov w13, s17 -; CHECK-GI-NEXT: ushll v26.4s, v7.4h, #0 -; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0 -; CHECK-GI-NEXT: addv s18, v18.4s -; CHECK-GI-NEXT: addv s5, v5.4s -; CHECK-GI-NEXT: sshll v27.4s, v21.4h, #0 -; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: sshll2 v21.4s, v21.8h, #0 -; CHECK-GI-NEXT: mul v6.4s, v6.4s, v20.4s -; CHECK-GI-NEXT: add w11, w12, w13 -; CHECK-GI-NEXT: fmov w12, s4 -; CHECK-GI-NEXT: addv s19, v19.4s -; CHECK-GI-NEXT: fmov w13, s5 -; CHECK-GI-NEXT: mul v20.4s, v26.4s, v27.4s -; CHECK-GI-NEXT: fmov w14, s0 -; CHECK-GI-NEXT: mul v7.4s, v7.4s, v21.4s -; CHECK-GI-NEXT: add w11, w11, w12 -; CHECK-GI-NEXT: fmov w12, s18 -; CHECK-GI-NEXT: addv s6, v6.4s -; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: add w9, w9, w14 -; CHECK-GI-NEXT: addv s1, v20.4s +; CHECK-GI-NEXT: addv s3, v18.4s +; CHECK-GI-NEXT: addv s5, v19.4s +; CHECK-GI-NEXT: fmov w10, s2 +; CHECK-GI-NEXT: add w8, w8, w9 +; CHECK-GI-NEXT: fmov w9, s4 +; CHECK-GI-NEXT: fmov w11, s3 ; CHECK-GI-NEXT: add w8, w8, w9 -; CHECK-GI-NEXT: add w12, w12, w13 -; CHECK-GI-NEXT: fmov w13, s19 -; CHECK-GI-NEXT: addv s2, v7.4s -; CHECK-GI-NEXT: add w11, w12, w13 -; CHECK-GI-NEXT: fmov w12, s6 -; CHECK-GI-NEXT: fmov w13, s2 -; CHECK-GI-NEXT: add w11, w11, w12 -; CHECK-GI-NEXT: fmov w12, s1 ; CHECK-GI-NEXT: add w10, w10, w11 -; CHECK-GI-NEXT: add w11, w12, w13 +; CHECK-GI-NEXT: fmov w11, s5 ; CHECK-GI-NEXT: add w9, w10, w11 ; CHECK-GI-NEXT: add w0, w8, w9 ; CHECK-GI-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload ; CHECK-GI-NEXT: ret entry: - %az = zext <48 x i8> %a to <48 x i32> - %bz = sext <48 x i8> %b to <48 x i32> - %m1 = mul nuw nsw <48 x i32> %az, %bz - %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m1) - %cz = zext <48 x i8> %c to <48 x i32> - %dz = sext <48 x i8> %d to <48 x i32> - %m2 = mul nuw nsw <48 x i32> %cz, %dz - %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %m2) + %az = sext <48 x i8> %a to <48 x i32> + %r1 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %az) + %cz = sext <48 x i8> %c to <48 x i32> + %r2 = call i32 @llvm.vector.reduce.add.v48i32(<48 x i32> %cz) %x = add i32 %r1, %r2 ret i32 %x }