[LoopVectorizePass] Polyhedron/linpk 40% regression on neoverse-v1 after #141865

[vzakhari]

Performance comes back when reverting 7ef77eb

Compilation command: flang -Ofast -mcpu=neoverse-v1 linpk.f90 -frtlib-add-rpath

Before the change:

           2124.44 msec task-clock                       #    0.997 CPUs utilized
        5510839431      cycles                           #    2.594 GHz
        6419270661      instructions                     #    1.16  insn per cycle

After the change:

           3101.06 msec task-clock                       #    0.997 CPUs utilized
        8045294105      cycles                           #    2.594 GHz
        5452469574      instructions                     #    0.68  insn per cycle

Attaching LLVM IR dumps before and after the change, and the assembly.

before.s.gz

before.log.gz

after.s.gz

after.log.gz

[vzakhari]

Hello @lukel97, can you please check if the changes are expected?

[lukel97]

Thanks for reporting, will take a look. The intention with 7ef77eb was not to have any significant changes on AArch64, maybe a factor 4 interleave isn't getting lowered properly

[lukel97]

It looks like now we're generating ld4/st4 whereas previously it emitted a pair of ld2/st2s:

 .LBB4_15:
-	ld2d	{ z3.d, z4.d }, p0/z, [x8]
+	ld4d	{ z2.d - z5.d }, p0/z, [x8]
 	subs	x16, x16, x11
-	ld2d	{ z5.d, z6.d }, p0/z, [x8, #2, mul vl]
-	ld2d	{ z16.d, z17.d }, p0/z, [x14]
-	movprfx	z18, z3
-	fmla	z18.d, p0/m, z16.d, z2.d
-	movprfx	z19, z4
-	fmla	z19.d, p0/m, z17.d, z2.d
-	ld2d	{ z3.d, z4.d }, p0/z, [x14, #2, mul vl]
+	ld4d	{ z16.d - z19.d }, p0/z, [x14]
 	add	x14, x14, x15
-	movprfx	z16, z5
-	fmla	z16.d, p0/m, z3.d, z1.d
-	movprfx	z17, z6
-	fmla	z17.d, p0/m, z4.d, z1.d
-	st2d	{ z18.d, z19.d }, p0, [x8]
-	st2d	{ z16.d, z17.d }, p0, [x8, #2, mul vl]
+	movprfx	z20, z2
+	fmla	z20.d, p0/m, z16.d, z1.d
+	movprfx	z21, z3
+	fmla	z21.d, p0/m, z17.d, z1.d
+	movprfx	z22, z4
+	fmla	z22.d, p0/m, z18.d, z1.d
+	movprfx	z23, z5
+	fmla	z23.d, p0/m, z19.d, z1.d
+	st4d	{ z20.d - z23.d }, p0, [x8]

I presume doing two ld2/st2s unlocks more ILP somehow on the neoverse-v1. But I'm surprised it's choosing to use an interleave group of 4, because looking at AArch64Subtarget.cpp MaxInterleaveFactor for NeoverseV1 should indeed be 2. Turns out TTI::getMaxInterleaveFactor != TLI::getMaxSupportedInterleaveFactor

Will investigate further tomorrow

[lukel97]

I think what's happening here is previously when the loop vectorizer emitted a series of interleaved @llvm.vector.interleave2 intrinsics, the ComplexDeinterleavingPass actually kicked in and removed the inner layer of interleaves:

  %strided.vec88 = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %205)
  %206 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec87, 0
  %207 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec88, 0
  %208 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec87, 1
  %209 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec88, 1
  %210 = getelementptr double, ptr %67, i64 %202
  %wide.vec89 = load <vscale x 8 x double>, ptr %210, align 8, !tbaa !74
  %strided.vec90 = call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec89)
  %211 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %strided.vec90, 0
  %212 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %strided.vec90, 1
  %strided.vec91 = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %211)
  %strided.vec92 = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.vector.deinterleave2.nxv4f64(<vscale x 4 x double> %212)
  %213 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec91, 0
  %214 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec92, 0
  %215 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec91, 1
  %216 = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } %strided.vec92, 1
  %217 = fmul fast <vscale x 2 x double> %213, %broadcast.splat
  %218 = fadd fast <vscale x 2 x double> %217, %206
  %219 = fmul fast <vscale x 2 x double> %214, %broadcast.splat
  %220 = fadd fast <vscale x 2 x double> %219, %207
  %221 = fmul fast <vscale x 2 x double> %215, %broadcast.splat
  %222 = fadd fast <vscale x 2 x double> %221, %208
  %223 = fmul fast <vscale x 2 x double> %216, %broadcast.splat
  %224 = fadd fast <vscale x 2 x double> %223, %209
  %interleaved.vec = call <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %218, <vscale x 2 x double> %222)
  %interleaved.vec93 = call <vscale x 4 x double> @llvm.vector.interleave2.nxv4f64(<vscale x 2 x double> %220, <vscale x 2 x double> %224)
  %interleaved.vec94 = call <vscale x 8 x double> @llvm.vector.interleave2.nxv8f64(<vscale x 4 x double> %interleaved.vec, <vscale x 4 x double> %interleaved.vec93)
  store <vscale x 8 x double> %interleaved.vec94, ptr %203, align 8, !tbaa !68
  %index.next = add nuw i64 %index, %197
  %225 = icmp eq i64 %index.next, %n.vec
  br i1 %225, label %middle.block, label %vector.body, !llvm.loop !76

vector.body:                                      ; preds = %vector.body, %vector.ph
  %lsr.iv179 = phi i64 [ %lsr.iv.next180, %vector.body ], [ %n.vec, %vector.ph ]
  %lsr.iv177 = phi ptr [ %scevgep178, %vector.body ], [ %scevgep176, %vector.ph ]
  %lsr.iv172 = phi ptr [ %scevgep173, %vector.body ], [ %scevgep171, %vector.ph ]
  %wide.vec = load <vscale x 8 x double>, ptr %lsr.iv172, align 8, !tbaa !70
  %strided.vec = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec)
  %280 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %strided.vec, 0
  %281 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %strided.vec, 1
  %wide.vec89 = load <vscale x 8 x double>, ptr %lsr.iv177, align 8, !tbaa !76
  %strided.vec90 = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec89)
  %282 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %strided.vec90, 0
  %283 = extractvalue { <vscale x 4 x double>, <vscale x 4 x double> } %strided.vec90, 1
  %284 = fmul fast <vscale x 4 x double> %282, %276
  %285 = fadd fast <vscale x 4 x double> %284, %280
  %286 = fmul fast <vscale x 4 x double> %283, %276
  %287 = fadd fast <vscale x 4 x double> %286, %281
  %interleaved.vec94 = tail call <vscale x 8 x double> @llvm.vector.interleave2.nxv8f64(<vscale x 4 x double> %285, <vscale x 4 x double> %287)
  store <vscale x 8 x double> %interleaved.vec94, ptr %lsr.iv172, align 8, !tbaa !70
  %scevgep173 = getelementptr i8, ptr %lsr.iv172, i64 %26
  %scevgep178 = getelementptr i8, ptr %lsr.iv177, i64 %26
  %lsr.iv.next180 = sub i64 %lsr.iv179, %28
  %288 = icmp eq i64 %lsr.iv.next180, 0
  br i1 %288, label %middle.block, label %vector.body, !llvm.loop !78

The InterleavedAccessPass then independently recognised these as two wide interleaves, which generated 2 ld2d/st2d each.

I'm not sure what the best path forward is. The ld4d code seems somewhat more 'canonical' and it's less instructions, it may just be a coincidence that the ld2d version is more favourable to the neoverse-v1's uarch. Do neoverse-v2/3 see the same regression?

One option is that we could teach ComplexDeinterleavingPass to treat [de]interleave4 intrinsics as multiple [de]interleave2s, but that's assuming that no uarch prefers ld4d/st4d.

[fhahn]

Is this a case were no interleaving would be required at all?

[lukel97]

Is this a case were no interleaving would be required at all?

I think so actually. It doesn't look like the interleave groups cross within themselves:

    INTERLEAVE-GROUP with factor 4 at %129, ir<%128>
      ir<%129> = load from index 0
      ir<%135> = load from index 1
      ir<%142> = load from index 2
      ir<%149> = load from index 3
    CLONE ir<%130> = getelementptr ir<%52>, ir<%127>
    INTERLEAVE-GROUP with factor 4 at %131, ir<%130>
      ir<%131> = load from index 0
      ir<%137> = load from index 1
      ir<%144> = load from index 2
      ir<%151> = load from index 3
    WIDEN ir<%132> = fmul fast ir<%131>, ir<%98>
    WIDEN ir<%133> = fadd fast ir<%132>, ir<%129>
    WIDEN ir<%138> = fmul fast ir<%137>, ir<%98>
    WIDEN ir<%139> = fadd fast ir<%138>, ir<%135>
    WIDEN ir<%145> = fmul fast ir<%144>, ir<%98>
    WIDEN ir<%146> = fadd fast ir<%145>, ir<%142>
    WIDEN ir<%152> = fmul fast ir<%151>, ir<%98>
    WIDEN ir<%153> = fadd fast ir<%152>, ir<%149>
    INTERLEAVE-GROUP with factor 4 at <badref>, ir<%128>
      store ir<%133> to index 0
      store ir<%139> to index 1
      store ir<%146> to index 2
      store ir<%153> to index 3

So this could just be

    WIDEN ir<%load1> = load ir<%128>
    WIDEN ir<%load2> = load ir<%130>
    WIDEN ir<%fmul> = fmul fast ir<%load1>, ir<%98>
    WIDEN ir<%fadd> = fadd fast ir<%fmul>, ir<%load2>
    WIDEN store ir<%fadd>, ir<%128>

[lukel97]

I think the correct fix for this is either a form of loop-aware SLP, or loop rerolling. Here's a much simplified version of the input IR:

define void @f(ptr noalias %p) {
entry:
  br label %loop

loop:
  %iv = phi i64 [0, %entry], [%iv.next, %loop]

  %iv0 = add nuw nsw i64 %iv, 0
  %gep0 = getelementptr i64, ptr %p, i64 %iv0
  %load0 = load i64, ptr %gep0
  %add0 = add i64 %load0, 1
  store i64 %add0, ptr %gep0

  %iv1 = add nuw nsw i64 %iv, 1
  %gep1 = getelementptr i64, ptr %p, i64 %iv1
  %load1 = load i64, ptr %gep1
  %add1 = add i64 %load1, 1
  store i64 %add1, ptr %gep1

  %iv.next = add nuw nsw i64 %iv, 2
  %done = icmp eq i64 %iv.next, 1024
  br i1 %done, label %exit, label %loop

exit:
  ret void
}

Today we generate an interleave group for this.

Unfortunately loop rerolling has been removed since it was unmaintained: #80972