Revert "cmd/compile: use generated loops instead of DUFFCOPY on amd64"

This reverts commit ec9e117 (CL 678620).

Reason for revert: causing regalloc to get into an infinite loop

Change-Id: Ie53c58c6126804af6d6883ea4acdcfb632a172bd
Reviewed-on: https://go-review.googlesource.com/c/go/+/695196
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Auto-Submit: Keith Randall <[email protected]>
Reviewed-by: Keith Randall <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: randall77

src/cmd/compile/internal/amd64/ssa.go

@@ -142,6 +142,45 @@ func memIdx(a *obj.Addr, v *ssa.Value) {
 	a.Index = i
 }
 
+// DUFFZERO consists of repeated blocks of 4 MOVUPSs + LEAQ,
+// See runtime/mkduff.go.
+const (
+	dzBlocks    = 16 // number of MOV/ADD blocks
+	dzBlockLen  = 4  // number of clears per block
+	dzBlockSize = 23 // size of instructions in a single block
+	dzMovSize   = 5  // size of single MOV instruction w/ offset
+	dzLeaqSize  = 4  // size of single LEAQ instruction
+	dzClearStep = 16 // number of bytes cleared by each MOV instruction
+)
+
+func duffStart(size int64) int64 {
+	x, _ := duff(size)
+	return x
+}
+func duffAdj(size int64) int64 {
+	_, x := duff(size)
+	return x
+}
+
+// duff returns the offset (from duffzero, in bytes) and pointer adjust (in bytes)
+// required to use the duffzero mechanism for a block of the given size.
+func duff(size int64) (int64, int64) {
+	if size < 32 || size > 1024 || size%dzClearStep != 0 {
+		panic("bad duffzero size")
+	}
+	steps := size / dzClearStep
+	blocks := steps / dzBlockLen
+	steps %= dzBlockLen
+	off := dzBlockSize * (dzBlocks - blocks)
+	var adj int64
+	if steps != 0 {
+		off -= dzLeaqSize
+		off -= dzMovSize * steps
+		adj -= dzClearStep * (dzBlockLen - steps)
+	}
+	return off, adj
+}
+
 func getgFromTLS(s *ssagen.State, r int16) {
 	// See the comments in cmd/internal/obj/x86/obj6.go
 	// near CanUse1InsnTLS for a detailed explanation of these instructions.
@@ -1065,110 +1104,20 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
 			zero16(off + n - 16)
 		}
 
-	case ssa.OpAMD64LoweredMove:
-		dstReg := v.Args[0].Reg()
-		srcReg := v.Args[1].Reg()
-		if dstReg == srcReg {
-			break
-		}
-		tmpReg := int16(x86.REG_X14)
-		n := v.AuxInt
-		if n < 16 {
-			v.Fatalf("Move too small %d", n)
-		}
-		// move 16 bytes from srcReg+off to dstReg+off.
-		move16 := func(off int64) {
-			move16(s, srcReg, dstReg, tmpReg, off)
-		}
-
-		// Generate copying instructions.
-		var off int64
-		for n >= 16 {
-			move16(off)
-			off += 16
-			n -= 16
-		}
-		if n != 0 {
-			// use partially overlapped read/write.
-			// TODO: use smaller operations when we can?
-			move16(off + n - 16)
-		}
-
-	case ssa.OpAMD64LoweredMoveLoop:
-		dstReg := v.Args[0].Reg()
-		srcReg := v.Args[1].Reg()
-		if dstReg == srcReg {
-			break
-		}
-		countReg := v.RegTmp()
-		tmpReg := int16(x86.REG_X14)
-		n := v.AuxInt
-		loopSize := int64(64)
-		if n < 3*loopSize {
-			// - a loop count of 0 won't work.
-			// - a loop count of 1 is useless.
-			// - a loop count of 2 is a code size ~tie
-			//     4 instructions to implement the loop
-			//     4 instructions in the loop body
-			//   vs
-			//     8 instructions in the straightline code
-			//   Might as well use straightline code.
-			v.Fatalf("ZeroLoop size too small %d", n)
-		}
-		// move 16 bytes from srcReg+off to dstReg+off.
-		move16 := func(off int64) {
-			move16(s, srcReg, dstReg, tmpReg, off)
-		}
-
-		// Put iteration count in a register.
-		//   MOVL    $n, countReg
-		p := s.Prog(x86.AMOVL)
-		p.From.Type = obj.TYPE_CONST
-		p.From.Offset = n / loopSize
-		p.To.Type = obj.TYPE_REG
-		p.To.Reg = countReg
-		cntInit := p
-
-		// Copy loopSize bytes starting at srcReg to dstReg.
-		for i := range loopSize / 16 {
-			move16(i * 16)
-		}
-		//   ADDQ    $loopSize, srcReg
-		p = s.Prog(x86.AADDQ)
-		p.From.Type = obj.TYPE_CONST
-		p.From.Offset = loopSize
-		p.To.Type = obj.TYPE_REG
-		p.To.Reg = srcReg
-		//   ADDQ    $loopSize, dstReg
-		p = s.Prog(x86.AADDQ)
-		p.From.Type = obj.TYPE_CONST
-		p.From.Offset = loopSize
-		p.To.Type = obj.TYPE_REG
-		p.To.Reg = dstReg
-		//   DECL    countReg
-		p = s.Prog(x86.ADECL)
-		p.To.Type = obj.TYPE_REG
-		p.To.Reg = countReg
-		// Jump to loop header if we're not done yet.
-		//   JNE     head
-		p = s.Prog(x86.AJNE)
-		p.To.Type = obj.TYPE_BRANCH
-		p.To.SetTarget(cntInit.Link)
-
-		// Multiples of the loop size are now done.
-		n %= loopSize
-
-		// Copy any fractional portion.
-		var off int64
-		for n >= 16 {
-			move16(off)
-			off += 16
-			n -= 16
-		}
-		if n != 0 {
-			// Use partially-overlapping copy.
-			move16(off + n - 16)
+	case ssa.OpAMD64DUFFCOPY:
+		p := s.Prog(obj.ADUFFCOPY)
+		p.To.Type = obj.TYPE_ADDR
+		p.To.Sym = ir.Syms.Duffcopy
+		if v.AuxInt%16 != 0 {
+			v.Fatalf("bad DUFFCOPY AuxInt %v", v.AuxInt)
 		}
+		p.To.Offset = 14 * (64 - v.AuxInt/16)
+		// 14 and 64 are magic constants.  14 is the number of bytes to encode:
+		//	MOVUPS	(SI), X0
+		//	ADDQ	$16, SI
+		//	MOVUPS	X0, (DI)
+		//	ADDQ	$16, DI
+		// and 64 is the number of such blocks. See src/runtime/duff_amd64.s:duffcopy.
 
 	case ssa.OpCopy: // TODO: use MOVQreg for reg->reg copies instead of OpCopy?
 		if v.Type.IsMemory() {
@@ -1760,21 +1709,3 @@ func zero16(s *ssagen.State, reg int16, off int64) {
 	p.To.Reg = reg
 	p.To.Offset = off
 }
-
-// move 16 bytes from src+off to dst+off using temporary register tmp.
-func move16(s *ssagen.State, src, dst, tmp int16, off int64) {
-	//   MOVUPS  off(srcReg), tmpReg
-	//   MOVUPS  tmpReg, off(dstReg)
-	p := s.Prog(x86.AMOVUPS)
-	p.From.Type = obj.TYPE_MEM
-	p.From.Reg = src
-	p.From.Offset = off
-	p.To.Type = obj.TYPE_REG
-	p.To.Reg = tmp
-	p = s.Prog(x86.AMOVUPS)
-	p.From.Type = obj.TYPE_REG
-	p.From.Reg = tmp
-	p.To.Type = obj.TYPE_MEM
-	p.To.Reg = dst
-	p.To.Offset = off
-}

src/cmd/compile/internal/ssa/_gen/AMD64.rules

@@ -264,6 +264,24 @@
 (Move [8] dst src mem) => (MOVQstore dst (MOVQload src mem) mem)
 (Move [16] dst src mem) => (MOVOstore dst (MOVOload src mem) mem)
 
+(Move [32] dst src mem) =>
+	(Move [16]
+		(OffPtr <dst.Type> dst [16])
+		(OffPtr <src.Type> src [16])
+		(Move [16] dst src mem))
+
+(Move [48] dst src mem) =>
+	(Move [32]
+		(OffPtr <dst.Type> dst [16])
+		(OffPtr <src.Type> src [16])
+		(Move [16] dst src mem))
+
+(Move [64] dst src mem) =>
+	(Move [32]
+		(OffPtr <dst.Type> dst [32])
+		(OffPtr <src.Type> src [32])
+		(Move [32] dst src mem))
+
 (Move [3] dst src mem) =>
 	(MOVBstore [2] dst (MOVBload [2] src mem)
 		(MOVWstore dst (MOVWload src mem) mem))
@@ -292,19 +310,28 @@
 	(MOVQstore [int32(s-8)] dst (MOVQload [int32(s-8)] src mem)
 		(MOVQstore dst (MOVQload src mem) mem))
 
-// Copying up to 192 bytes uses straightline code.
-(Move [s] dst src mem) && s > 16 && s < 192 && logLargeCopy(v, s) => (LoweredMove [s] dst src mem)
-
-// Copying up to ~1KB uses a small loop.
-(Move [s] dst src mem) && s >= 192 && s <= repMoveThreshold && logLargeCopy(v, s) => (LoweredMoveLoop [s] dst src mem)
+// Adjust moves to be a multiple of 16 bytes.
+(Move [s] dst src mem)
+	&& s > 16 && s%16 != 0 && s%16 <= 8 =>
+	(Move [s-s%16]
+		(OffPtr <dst.Type> dst [s%16])
+		(OffPtr <src.Type> src [s%16])
+		(MOVQstore dst (MOVQload src mem) mem))
+(Move [s] dst src mem)
+	&& s > 16 && s%16 != 0 && s%16 > 8 =>
+	(Move [s-s%16]
+		(OffPtr <dst.Type> dst [s%16])
+		(OffPtr <src.Type> src [s%16])
+		(MOVOstore dst (MOVOload src mem) mem))
+
+// Medium copying uses a duff device.
+(Move [s] dst src mem)
+	&& s > 64 && s <= 16*64 && s%16 == 0
+	&& logLargeCopy(v, s) =>
+	(DUFFCOPY [s] dst src mem)
 
 // Large copying uses REP MOVSQ.
-(Move [s] dst src mem) && s > repMoveThreshold && s%8 != 0 =>
-	(Move [s-s%8]
-		(OffPtr <dst.Type> dst [s%8])
-		(OffPtr <src.Type> src [s%8])
-		(MOVQstore dst (MOVQload src mem) mem))
-(Move [s] dst src mem) && s > repMoveThreshold && s%8 == 0 && logLargeCopy(v, s) =>
+(Move [s] dst src mem) && s > 16*64 && s%8 == 0 && logLargeCopy(v, s) =>
 	(REPMOVSQ dst src (MOVQconst [s/8]) mem)
 
 // Lowering Zero instructions

src/cmd/compile/internal/ssa/_gen/AMD64Ops.go

@@ -939,38 +939,20 @@ func init() {
 		// arg0 = destination pointer
 		// arg1 = source pointer
 		// arg2 = mem
-		// auxint = # of bytes to copy
+		// auxint = # of bytes to copy, must be multiple of 16
 		// returns memory
 		{
-			name:      "LoweredMove",
+			name:      "DUFFCOPY",
 			aux:       "Int64",
 			argLength: 3,
 			reg: regInfo{
-				inputs:   []regMask{gp, gp},
-				clobbers: buildReg("X14"), // uses X14 as a temporary
+				inputs:   []regMask{buildReg("DI"), buildReg("SI")},
+				clobbers: buildReg("DI SI X0"), // uses X0 as a temporary
 			},
-			faultOnNilArg0: true,
-			faultOnNilArg1: true,
-		},
-		// arg0 = destination pointer
-		// arg1 = source pointer
-		// arg2 = mem
-		// auxint = # of bytes to copy
-		// returns memory
-		{
-			name:      "LoweredMoveLoop",
-			aux:       "Int64",
-			argLength: 3,
-			reg: regInfo{
-				inputs:       []regMask{gp, gp},
-				clobbers:     buildReg("X14"), // uses X14 as a temporary
-				clobbersArg0: true,
-				clobbersArg1: true,
-			},
-			clobberFlags:   true,
-			faultOnNilArg0: true,
-			faultOnNilArg1: true,
-			needIntTemp:    true,
+			clobberFlags: true,
+			//faultOnNilArg0: true, // Note: removed for 73748. TODO: reenable at some point
+			//faultOnNilArg1: true,
+			unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts
 		},
 
 		// arg0 = destination pointer

src/cmd/compile/internal/ssa/opGen.go

@@ -561,14 +561,7 @@ func (s *regAllocState) allocValToReg(v *Value, mask regMask, nospill bool, pos
 	pos = pos.WithNotStmt()
 	// Check if v is already in a requested register.
 	if mask&vi.regs != 0 {
-		mask &= vi.regs
-		r := pickReg(mask)
-		if mask.contains(s.SPReg) {
-			// Prefer the stack pointer if it is allowed.
-			// (Needed because the op might have an Aux symbol
-			// that needs SP as its base.)
-			r = s.SPReg
-		}
+		r := pickReg(mask & vi.regs)
 		if !s.allocatable.contains(r) {
 			return v // v is in a fixed register
 		}