[AArch64][SVE] Improve VLS imm addressing modes.

When compiling VLS SVE, the compiler often replaces VL-based offsets
with immediate-based ones. This leads to a mismatch during isel since
SVE loads/stores generally expect immediate offsets relative to VL.

For example, given:
```c

svfloat64_t foo(const double *x) {
  svbool_t pg = svptrue_b64();
  return svld1_f64(pg, x+svcntd());
}
```

When compiled with `-msve-vector-bits=128`, we currently generate:
```gas
foo:
        ptrue   p0.d
        mov     x8, #2
        ld1d    { z0.d }, p0/z, [x0, x8, lsl #3]
        ret
```

In practice, we could instead be generating:
```gas
foo:
        ldr     z0, [x0, #1, mul vl]
        ret
```

Likewise for other types, stores, and other VLS lengths.

Author: rj-jesus

clang/test/CodeGen/AArch64/sve-vector-bits-codegen.c

@@ -13,12 +13,9 @@
 
 void func(int *restrict a, int *restrict b) {
 // CHECK-LABEL: func
-// CHECK256-COUNT-1: str
-// CHECK256-COUNT-7: st1w
-// CHECK512-COUNT-1: str
-// CHECK512-COUNT-3: st1w
-// CHECK1024-COUNT-1: str
-// CHECK1024-COUNT-1: st1w
+// CHECK256-COUNT-8: str
+// CHECK512-COUNT-4: str
+// CHECK1024-COUNT-2: str
 // CHECK2048-COUNT-1: st1w
 #pragma clang loop vectorize(enable)
   for (int i = 0; i < 64; ++i)

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

@@ -7379,13 +7379,27 @@ bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
   if (N.getOpcode() != ISD::ADD)
     return false;
 
-  SDValue VScale = N.getOperand(1);
-  if (VScale.getOpcode() != ISD::VSCALE)
+  int64_t MulImm = std::numeric_limits<int64_t>::max();
+  if (SDValue VScale = N.getOperand(1); VScale.getOpcode() == ISD::VSCALE)
+    MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
+  else if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
+    int64_t ByteOffset = C->getSExtValue();
+    constexpr auto SVEBitsPerBlock = AArch64::SVEBitsPerBlock;
+    auto MinVScale = Subtarget->getMinSVEVectorSizeInBits() / SVEBitsPerBlock;
+    auto MaxVScale = Subtarget->getMaxSVEVectorSizeInBits() / SVEBitsPerBlock;
+
+    if (!MaxVScale || MinVScale != MaxVScale || ByteOffset % MaxVScale != 0)
+      return false;
+
+    MulImm = ByteOffset / MaxVScale;
+  } else
     return false;
 
+  assert(MulImm != std::numeric_limits<int64_t>::max() &&
+         "Uninitialized MulImm.");
+
   TypeSize TS = MemVT.getSizeInBits();
   int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
-  int64_t MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
 
   if ((MulImm % MemWidthBytes) != 0)
     return false;

llvm/test/CodeGen/AArch64/sve-fixed-length-offsets.ll

@@ -17,42 +17,32 @@ define void @nxv16i8(ptr %ldptr, ptr %stptr) {
 ;
 ; CHECK-128-LABEL: nxv16i8:
 ; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ptrue p0.b
-; CHECK-128-NEXT:    mov w8, #256 // =0x100
-; CHECK-128-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
-; CHECK-128-NEXT:    st1b { z0.b }, p0, [x1, x8]
+; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
+; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
 ; CHECK-128-NEXT:    ret
 ;
 ; CHECK-256-LABEL: nxv16i8:
 ; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ptrue p0.b
-; CHECK-256-NEXT:    mov w8, #256 // =0x100
-; CHECK-256-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
-; CHECK-256-NEXT:    st1b { z0.b }, p0, [x1, x8]
+; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
+; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
 ; CHECK-256-NEXT:    ret
 ;
 ; CHECK-512-LABEL: nxv16i8:
 ; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ptrue p0.b
-; CHECK-512-NEXT:    mov w8, #256 // =0x100
-; CHECK-512-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
-; CHECK-512-NEXT:    st1b { z0.b }, p0, [x1, x8]
+; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
+; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
 ; CHECK-512-NEXT:    ret
 ;
 ; CHECK-1024-LABEL: nxv16i8:
 ; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ptrue p0.b
-; CHECK-1024-NEXT:    mov w8, #256 // =0x100
-; CHECK-1024-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
-; CHECK-1024-NEXT:    st1b { z0.b }, p0, [x1, x8]
+; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
+; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
 ; CHECK-1024-NEXT:    ret
 ;
 ; CHECK-2048-LABEL: nxv16i8:
 ; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.b
-; CHECK-2048-NEXT:    mov w8, #256 // =0x100
-; CHECK-2048-NEXT:    ld1b { z0.b }, p0/z, [x0, x8]
-; CHECK-2048-NEXT:    st1b { z0.b }, p0, [x1, x8]
+; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
+; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
 ; CHECK-2048-NEXT:    ret
   %ldoff = getelementptr inbounds nuw i8, ptr %ldptr, i64 256
   %stoff = getelementptr inbounds nuw i8, ptr %stptr, i64 256
@@ -72,42 +62,32 @@ define void @nxv8i16(ptr %ldptr, ptr %stptr) {
 ;
 ; CHECK-128-LABEL: nxv8i16:
 ; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ptrue p0.h
-; CHECK-128-NEXT:    mov x8, #128 // =0x80
-; CHECK-128-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-128-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
+; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
 ; CHECK-128-NEXT:    ret
 ;
 ; CHECK-256-LABEL: nxv8i16:
 ; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ptrue p0.h
-; CHECK-256-NEXT:    mov x8, #128 // =0x80
-; CHECK-256-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-256-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
+; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
 ; CHECK-256-NEXT:    ret
 ;
 ; CHECK-512-LABEL: nxv8i16:
 ; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ptrue p0.h
-; CHECK-512-NEXT:    mov x8, #128 // =0x80
-; CHECK-512-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-512-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
+; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
 ; CHECK-512-NEXT:    ret
 ;
 ; CHECK-1024-LABEL: nxv8i16:
 ; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ptrue p0.h
-; CHECK-1024-NEXT:    mov x8, #128 // =0x80
-; CHECK-1024-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-1024-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
+; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
 ; CHECK-1024-NEXT:    ret
 ;
 ; CHECK-2048-LABEL: nxv8i16:
 ; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.h
-; CHECK-2048-NEXT:    mov x8, #128 // =0x80
-; CHECK-2048-NEXT:    ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-2048-NEXT:    st1h { z0.h }, p0, [x1, x8, lsl #1]
+; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
+; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
 ; CHECK-2048-NEXT:    ret
   %ldoff = getelementptr inbounds nuw i16, ptr %ldptr, i64 128
   %stoff = getelementptr inbounds nuw i16, ptr %stptr, i64 128
@@ -127,42 +107,32 @@ define void @nxv4i32(ptr %ldptr, ptr %stptr) {
 ;
 ; CHECK-128-LABEL: nxv4i32:
 ; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ptrue p0.s
-; CHECK-128-NEXT:    mov x8, #64 // =0x40
-; CHECK-128-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-128-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
+; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
 ; CHECK-128-NEXT:    ret
 ;
 ; CHECK-256-LABEL: nxv4i32:
 ; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ptrue p0.s
-; CHECK-256-NEXT:    mov x8, #64 // =0x40
-; CHECK-256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-256-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
+; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
 ; CHECK-256-NEXT:    ret
 ;
 ; CHECK-512-LABEL: nxv4i32:
 ; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ptrue p0.s
-; CHECK-512-NEXT:    mov x8, #64 // =0x40
-; CHECK-512-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-512-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
+; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
 ; CHECK-512-NEXT:    ret
 ;
 ; CHECK-1024-LABEL: nxv4i32:
 ; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ptrue p0.s
-; CHECK-1024-NEXT:    mov x8, #64 // =0x40
-; CHECK-1024-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-1024-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
+; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
 ; CHECK-1024-NEXT:    ret
 ;
 ; CHECK-2048-LABEL: nxv4i32:
 ; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.s
-; CHECK-2048-NEXT:    mov x8, #64 // =0x40
-; CHECK-2048-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
-; CHECK-2048-NEXT:    st1w { z0.s }, p0, [x1, x8, lsl #2]
+; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
+; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
 ; CHECK-2048-NEXT:    ret
   %ldoff = getelementptr inbounds nuw i32, ptr %ldptr, i64 64
   %stoff = getelementptr inbounds nuw i32, ptr %stptr, i64 64
@@ -182,42 +152,32 @@ define void @nxv2i64(ptr %ldptr, ptr %stptr) {
 ;
 ; CHECK-128-LABEL: nxv2i64:
 ; CHECK-128:       // %bb.0:
-; CHECK-128-NEXT:    ptrue p0.d
-; CHECK-128-NEXT:    mov x8, #32 // =0x20
-; CHECK-128-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-128-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; CHECK-128-NEXT:    ldr z0, [x0, #16, mul vl]
+; CHECK-128-NEXT:    str z0, [x1, #16, mul vl]
 ; CHECK-128-NEXT:    ret
 ;
 ; CHECK-256-LABEL: nxv2i64:
 ; CHECK-256:       // %bb.0:
-; CHECK-256-NEXT:    ptrue p0.d
-; CHECK-256-NEXT:    mov x8, #32 // =0x20
-; CHECK-256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-256-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; CHECK-256-NEXT:    ldr z0, [x0, #8, mul vl]
+; CHECK-256-NEXT:    str z0, [x1, #8, mul vl]
 ; CHECK-256-NEXT:    ret
 ;
 ; CHECK-512-LABEL: nxv2i64:
 ; CHECK-512:       // %bb.0:
-; CHECK-512-NEXT:    ptrue p0.d
-; CHECK-512-NEXT:    mov x8, #32 // =0x20
-; CHECK-512-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-512-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; CHECK-512-NEXT:    ldr z0, [x0, #4, mul vl]
+; CHECK-512-NEXT:    str z0, [x1, #4, mul vl]
 ; CHECK-512-NEXT:    ret
 ;
 ; CHECK-1024-LABEL: nxv2i64:
 ; CHECK-1024:       // %bb.0:
-; CHECK-1024-NEXT:    ptrue p0.d
-; CHECK-1024-NEXT:    mov x8, #32 // =0x20
-; CHECK-1024-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-1024-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; CHECK-1024-NEXT:    ldr z0, [x0, #2, mul vl]
+; CHECK-1024-NEXT:    str z0, [x1, #2, mul vl]
 ; CHECK-1024-NEXT:    ret
 ;
 ; CHECK-2048-LABEL: nxv2i64:
 ; CHECK-2048:       // %bb.0:
-; CHECK-2048-NEXT:    ptrue p0.d
-; CHECK-2048-NEXT:    mov x8, #32 // =0x20
-; CHECK-2048-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
-; CHECK-2048-NEXT:    st1d { z0.d }, p0, [x1, x8, lsl #3]
+; CHECK-2048-NEXT:    ldr z0, [x0, #1, mul vl]
+; CHECK-2048-NEXT:    str z0, [x1, #1, mul vl]
 ; CHECK-2048-NEXT:    ret
   %ldoff = getelementptr inbounds nuw i64, ptr %ldptr, i64 32
   %stoff = getelementptr inbounds nuw i64, ptr %stptr, i64 32

llvm/test/CodeGen/AArch64/sve-fixed-length-shuffles.ll

@@ -30,64 +30,64 @@ define void @crash_when_lowering_extract_shuffle(ptr %dst, i1 %cond) vscale_rang
 ; CHECK-NEXT:  // %bb.1: // %vector.body
 ; CHECK-NEXT:    mov z0.b, #0 // =0x0
 ; CHECK-NEXT:    ptrue p0.s
-; CHECK-NEXT:    mov x9, #8 // =0x8
-; CHECK-NEXT:    mov x10, #24 // =0x18
+; CHECK-NEXT:    mov x9, #24 // =0x18
 ; CHECK-NEXT:    umov w8, v0.b[8]
-; CHECK-NEXT:    mov v1.16b, v0.16b
-; CHECK-NEXT:    mov v1.b[1], v0.b[1]
-; CHECK-NEXT:    fmov s2, w8
-; CHECK-NEXT:    mov x8, #16 // =0x10
-; CHECK-NEXT:    mov v2.b[1], v0.b[9]
-; CHECK-NEXT:    mov v1.b[2], v0.b[2]
-; CHECK-NEXT:    mov v2.b[2], v0.b[10]
-; CHECK-NEXT:    mov v1.b[3], v0.b[3]
-; CHECK-NEXT:    mov v2.b[3], v0.b[11]
-; CHECK-NEXT:    mov v1.b[4], v0.b[4]
-; CHECK-NEXT:    mov v2.b[4], v0.b[12]
-; CHECK-NEXT:    mov v1.b[5], v0.b[5]
-; CHECK-NEXT:    mov v2.b[5], v0.b[13]
-; CHECK-NEXT:    mov v1.b[6], v0.b[6]
-; CHECK-NEXT:    mov v2.b[6], v0.b[14]
-; CHECK-NEXT:    mov v1.b[7], v0.b[7]
-; CHECK-NEXT:    mov v2.b[7], v0.b[15]
-; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #16
-; CHECK-NEXT:    uunpklo z1.h, z1.b
-; CHECK-NEXT:    ext v3.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    uunpklo z0.h, z0.b
+; CHECK-NEXT:    mov v2.16b, v0.16b
+; CHECK-NEXT:    mov z3.d, z0.d
+; CHECK-NEXT:    mov v2.b[1], v0.b[1]
+; CHECK-NEXT:    ext z3.b, z3.b, z0.b, #16
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    mov x8, #8 // =0x8
+; CHECK-NEXT:    ext v4.16b, v3.16b, v3.16b, #8
+; CHECK-NEXT:    mov v1.b[1], v0.b[9]
+; CHECK-NEXT:    mov v2.b[2], v0.b[2]
+; CHECK-NEXT:    mov v1.b[2], v0.b[10]
+; CHECK-NEXT:    mov v2.b[3], v0.b[3]
+; CHECK-NEXT:    mov v1.b[3], v0.b[11]
+; CHECK-NEXT:    mov v2.b[4], v0.b[4]
+; CHECK-NEXT:    mov v1.b[4], v0.b[12]
+; CHECK-NEXT:    mov v2.b[5], v0.b[5]
+; CHECK-NEXT:    mov v1.b[5], v0.b[13]
+; CHECK-NEXT:    mov v2.b[6], v0.b[6]
+; CHECK-NEXT:    mov v1.b[6], v0.b[14]
+; CHECK-NEXT:    mov v2.b[7], v0.b[7]
+; CHECK-NEXT:    mov v1.b[7], v0.b[15]
 ; CHECK-NEXT:    uunpklo z2.h, z2.b
-; CHECK-NEXT:    uunpklo z1.s, z1.h
-; CHECK-NEXT:    uunpklo z3.h, z3.b
-; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z0.h, z1.b
+; CHECK-NEXT:    uunpklo z1.h, z3.b
+; CHECK-NEXT:    uunpklo z3.h, z4.b
 ; CHECK-NEXT:    uunpklo z2.s, z2.h
-; CHECK-NEXT:    lsl z1.s, z1.s, #31
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    uunpklo z1.s, z1.h
 ; CHECK-NEXT:    uunpklo z3.s, z3.h
-; CHECK-NEXT:    lsl z0.s, z0.s, #31
-; CHECK-NEXT:    asr z1.s, z1.s, #31
 ; CHECK-NEXT:    lsl z2.s, z2.s, #31
-; CHECK-NEXT:    asr z0.s, z0.s, #31
-; CHECK-NEXT:    and z1.s, z1.s, #0x1
+; CHECK-NEXT:    lsl z0.s, z0.s, #31
+; CHECK-NEXT:    lsl z1.s, z1.s, #31
 ; CHECK-NEXT:    lsl z3.s, z3.s, #31
 ; CHECK-NEXT:    asr z2.s, z2.s, #31
-; CHECK-NEXT:    and z0.s, z0.s, #0x1
-; CHECK-NEXT:    cmpne p4.s, p0/z, z1.s, #0
-; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0]
+; CHECK-NEXT:    asr z0.s, z0.s, #31
+; CHECK-NEXT:    asr z1.s, z1.s, #31
 ; CHECK-NEXT:    asr z3.s, z3.s, #31
 ; CHECK-NEXT:    and z2.s, z2.s, #0x1
-; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
-; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK-NEXT:    and z0.s, z0.s, #0x1
+; CHECK-NEXT:    and z1.s, z1.s, #0x1
 ; CHECK-NEXT:    and z3.s, z3.s, #0x1
-; CHECK-NEXT:    cmpne p2.s, p0/z, z2.s, #0
-; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0, x9, lsl #2]
-; CHECK-NEXT:    mov z1.s, p4/m, #0 // =0x0
+; CHECK-NEXT:    cmpne p4.s, p0/z, z2.s, #0
+; CHECK-NEXT:    ld1w { z2.s }, p0/z, [x0]
+; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT:    cmpne p2.s, p0/z, z1.s, #0
 ; CHECK-NEXT:    cmpne p3.s, p0/z, z3.s, #0
-; CHECK-NEXT:    ld1w { z3.s }, p0/z, [x0, x10, lsl #2]
+; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
+; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, #1, mul vl]
+; CHECK-NEXT:    ld1w { z3.s }, p0/z, [x0, x9, lsl #2]
+; CHECK-NEXT:    mov z2.s, p4/m, #0 // =0x0
 ; CHECK-NEXT:    mov z0.s, p1/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.s, p2/m, #0 // =0x0
-; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
-; CHECK-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; CHECK-NEXT:    mov z1.s, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.s, p3/m, #0 // =0x0
-; CHECK-NEXT:    st1w { z2.s }, p0, [x0, x9, lsl #2]
-; CHECK-NEXT:    st1w { z3.s }, p0, [x0, x10, lsl #2]
+; CHECK-NEXT:    st1w { z2.s }, p0, [x0]
+; CHECK-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; CHECK-NEXT:    st1w { z1.s }, p0, [x0, #1, mul vl]
+; CHECK-NEXT:    st1w { z3.s }, p0, [x0, x9, lsl #2]
 ; CHECK-NEXT:  .LBB1_2: // %exit
 ; CHECK-NEXT:    ret
   %broadcast.splat = shufflevector <32 x i1> zeroinitializer, <32 x i1> zeroinitializer, <32 x i32> zeroinitializer