cmd/compile,runtime: remember idx+len for bounds check failure with less code

Currently we must put the index and length into specific registers so
we can call into the runtime to report a bounds check failure.

So a typical bounds check call is something like:

MOVD  R3, R0
MOVD  R7, R1
CALL  runtime.panicIndex

or, if for instance the index is constant,

MOVD  $7, R0
MOVD  R9, R1
CALL  runtime.panicIndex

Sometimes the MOVD can be avoided, if the value happens to be in the
right register already. But that's not terribly common, and doesn't
work at all for constants.

Let's get rid of those MOVD instructions. They pollute the instruction
cache and are almost never executed.

Instead, we'll encode in a PCDATA table where the runtime should find
the index and length. The table encodes, for each index and length,
whether it is a constant or in a register, and which register or
constant it is.

That way, we can avoid all those useless MOVDs. Instead, we can figure
out the index and length at runtime. This makes the bounds panic path
slower, but that's a good tradeoff.

We can encode registers 0-15 and constants 0-31. Anything outside that
range still needs to use an explicit instruction.

This CL is the foundation, followon CLs will move each architecture
to the new strategy.

Change-Id: I705c511e546e6aac59fed922a8eaed4585e96820
Reviewed-on: https://go-review.googlesource.com/c/go/+/682396
Reviewed-by: Michael Knyszek <[email protected]>
Reviewed-by: David Chase <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: randall77

src/cmd/compile/internal/ir/symtab.go

@@ -37,6 +37,8 @@ type symsStruct struct {
 	Msanmove          *obj.LSym
 	Newobject         *obj.LSym
 	Newproc           *obj.LSym
+	PanicBounds       *obj.LSym
+	PanicExtend       *obj.LSym
 	Panicdivide       *obj.LSym
 	Panicshift        *obj.LSym
 	PanicdottypeE     *obj.LSym

src/cmd/compile/internal/ssa/_gen/rulegen.go

@@ -540,6 +540,13 @@ func (u *unusedInspector) node(node ast.Node) {
 			}
 		}
 	case *ast.BasicLit:
+	case *ast.CompositeLit:
+		for _, e := range node.Elts {
+			u.node(e)
+		}
+	case *ast.KeyValueExpr:
+		u.node(node.Key)
+		u.node(node.Value)
 	case *ast.ValueSpec:
 		u.exprs(node.Values)
 	default:
@@ -1431,7 +1438,8 @@ func parseValue(val string, arch arch, loc string) (op opData, oparch, typ, auxi
 func opHasAuxInt(op opData) bool {
 	switch op.aux {
 	case "Bool", "Int8", "Int16", "Int32", "Int64", "Int128", "UInt8", "Float32", "Float64",
-		"SymOff", "CallOff", "SymValAndOff", "TypSize", "ARM64BitField", "FlagConstant", "CCop":
+		"SymOff", "CallOff", "SymValAndOff", "TypSize", "ARM64BitField", "FlagConstant", "CCop",
+		"PanicBoundsC", "PanicBoundsCC":
 		return true
 	}
 	return false
@@ -1440,7 +1448,7 @@ func opHasAuxInt(op opData) bool {
 func opHasAux(op opData) bool {
 	switch op.aux {
 	case "String", "Sym", "SymOff", "Call", "CallOff", "SymValAndOff", "Typ", "TypSize",
-		"S390XCCMask", "S390XRotateParams":
+		"S390XCCMask", "S390XRotateParams", "PanicBoundsC", "PanicBoundsCC":
 		return true
 	}
 	return false
@@ -1795,6 +1803,10 @@ func (op opData) auxType() string {
 		return "s390x.CCMask"
 	case "S390XRotateParams":
 		return "s390x.RotateParams"
+	case "PanicBoundsC":
+		return "PanicBoundsC"
+	case "PanicBoundsCC":
+		return "PanicBoundsCC"
 	default:
 		return "invalid"
 	}
@@ -1835,6 +1847,8 @@ func (op opData) auxIntType() string {
 		return "flagConstant"
 	case "ARM64BitField":
 		return "arm64BitField"
+	case "PanicBoundsC", "PanicBoundsCC":
+		return "int64"
 	default:
 		return "invalid"
 	}

src/cmd/compile/internal/ssa/check.go

@@ -215,6 +215,9 @@ func checkFunc(f *Func) {
 					f.Fatalf("bad FlagConstant AuxInt value for %v", v)
 				}
 				canHaveAuxInt = true
+			case auxPanicBoundsC, auxPanicBoundsCC:
+				canHaveAux = true
+				canHaveAuxInt = true
 			default:
 				f.Fatalf("unknown aux type for %s", v.Op)
 			}

src/cmd/compile/internal/ssa/op.go

@@ -6,10 +6,12 @@ package ssa
 
 import (
 	"cmd/compile/internal/abi"
+	"cmd/compile/internal/base"
 	"cmd/compile/internal/ir"
 	"cmd/compile/internal/types"
 	"cmd/internal/obj"
 	"fmt"
+	rtabi "internal/abi"
 	"strings"
 )
 
@@ -365,6 +367,9 @@ const (
 	auxCall                   // aux is a *ssa.AuxCall
 	auxCallOff                // aux is a *ssa.AuxCall, AuxInt is int64 param (in+out) size
 
+	auxPanicBoundsC  // constant for a bounds failure
+	auxPanicBoundsCC // two constants for a bounds failure
+
 	// architecture specific aux types
 	auxARM64BitField     // aux is an arm64 bitfield lsb and width packed into auxInt
 	auxS390XRotateParams // aux is a s390x rotate parameters object encoding start bit, end bit and rotate amount
@@ -523,6 +528,50 @@ func boundsABI(b int64) int {
 	}
 }
 
+// Returns the bounds error code needed by the runtime, and
+// whether the x field is signed.
+func (b BoundsKind) Code() (rtabi.BoundsErrorCode, bool) {
+	switch b {
+	case BoundsIndex:
+		return rtabi.BoundsIndex, true
+	case BoundsIndexU:
+		return rtabi.BoundsIndex, false
+	case BoundsSliceAlen:
+		return rtabi.BoundsSliceAlen, true
+	case BoundsSliceAlenU:
+		return rtabi.BoundsSliceAlen, false
+	case BoundsSliceAcap:
+		return rtabi.BoundsSliceAcap, true
+	case BoundsSliceAcapU:
+		return rtabi.BoundsSliceAcap, false
+	case BoundsSliceB:
+		return rtabi.BoundsSliceB, true
+	case BoundsSliceBU:
+		return rtabi.BoundsSliceB, false
+	case BoundsSlice3Alen:
+		return rtabi.BoundsSlice3Alen, true
+	case BoundsSlice3AlenU:
+		return rtabi.BoundsSlice3Alen, false
+	case BoundsSlice3Acap:
+		return rtabi.BoundsSlice3Acap, true
+	case BoundsSlice3AcapU:
+		return rtabi.BoundsSlice3Acap, false
+	case BoundsSlice3B:
+		return rtabi.BoundsSlice3B, true
+	case BoundsSlice3BU:
+		return rtabi.BoundsSlice3B, false
+	case BoundsSlice3C:
+		return rtabi.BoundsSlice3C, true
+	case BoundsSlice3CU:
+		return rtabi.BoundsSlice3C, false
+	case BoundsConvert:
+		return rtabi.BoundsConvert, false
+	default:
+		base.Fatalf("bad bounds kind %d", b)
+		return 0, false
+	}
+}
+
 // arm64BitField is the GO type of ARM64BitField auxInt.
 // if x is an ARM64BitField, then width=x&0xff, lsb=(x>>8)&0xff, and
 // width+lsb<64 for 64-bit variant, width+lsb<32 for 32-bit variant.

src/cmd/compile/internal/ssa/rewrite.go

@@ -2673,3 +2673,32 @@ func flagify(v *Value) bool {
 	v.AddArg(inner)
 	return true
 }
+
+// PanicBoundsC contains a constant for a bounds failure.
+type PanicBoundsC struct {
+	C int64
+}
+
+// PanicBoundsCC contains 2 constants for a bounds failure.
+type PanicBoundsCC struct {
+	Cx int64
+	Cy int64
+}
+
+func (p PanicBoundsC) CanBeAnSSAAux() {
+}
+func (p PanicBoundsCC) CanBeAnSSAAux() {
+}
+
+func auxToPanicBoundsC(i Aux) PanicBoundsC {
+	return i.(PanicBoundsC)
+}
+func auxToPanicBoundsCC(i Aux) PanicBoundsCC {
+	return i.(PanicBoundsCC)
+}
+func panicBoundsCToAux(p PanicBoundsC) Aux {
+	return p
+}
+func panicBoundsCCToAux(p PanicBoundsCC) Aux {
+	return p
+}

src/cmd/compile/internal/ssagen/ssa.go

@@ -137,6 +137,8 @@ func InitConfig() {
 	ir.Syms.Asanwrite = typecheck.LookupRuntimeFunc("asanwrite")
 	ir.Syms.Newobject = typecheck.LookupRuntimeFunc("newobject")
 	ir.Syms.Newproc = typecheck.LookupRuntimeFunc("newproc")
+	ir.Syms.PanicBounds = typecheck.LookupRuntimeFunc("panicBounds")
+	ir.Syms.PanicExtend = typecheck.LookupRuntimeFunc("panicExtend")
 	ir.Syms.Panicdivide = typecheck.LookupRuntimeFunc("panicdivide")
 	ir.Syms.PanicdottypeE = typecheck.LookupRuntimeFunc("panicdottypeE")
 	ir.Syms.PanicdottypeI = typecheck.LookupRuntimeFunc("panicdottypeI")

src/internal/abi/bounds.go

@@ -18,4 +18,96 @@ const (
 	BoundsSlice3B                           // s[?:x:y], 0 <= x <= y failed (but boundsSlice3A didn't happen)
 	BoundsSlice3C                           // s[x:y:?], 0 <= x <= y failed (but boundsSlice3A/B didn't happen)
 	BoundsConvert                           // (*[x]T)(s), 0 <= x <= len(s) failed
+	numBoundsCodes
 )
+
+const (
+	BoundsMaxReg   = 15
+	BoundsMaxConst = 31
+)
+
+// Here's how we encode PCDATA_PanicBounds entries:
+
+// We allow 16 registers (0-15) and 32 constants (0-31).
+// Encode the following constant c:
+//     bits    use
+// -----------------------------
+//       0     x is in a register
+//       1     y is in a register
+//
+// if x is in a register
+//       2     x is signed
+//     [3:6]   x's register number
+// else
+//     [2:6]   x's constant value
+//
+// if y is in a register
+//     [7:10]  y's register number
+// else
+//     [7:11]  y's constant value
+//
+// The final integer is c * numBoundsCode + code
+
+// TODO: 32-bit
+
+// Encode bounds failure information into an integer for PCDATA_PanicBounds.
+// Register numbers must be in 0-15. Constants must be in 0-31.
+func BoundsEncode(code BoundsErrorCode, signed, xIsReg, yIsReg bool, xVal, yVal int) int {
+	c := int(0)
+	if xIsReg {
+		c |= 1 << 0
+		if signed {
+			c |= 1 << 2
+		}
+		if xVal < 0 || xVal > BoundsMaxReg {
+			panic("bad xReg")
+		}
+		c |= xVal << 3
+	} else {
+		if xVal < 0 || xVal > BoundsMaxConst {
+			panic("bad xConst")
+		}
+		c |= xVal << 2
+	}
+	if yIsReg {
+		c |= 1 << 1
+		if yVal < 0 || yVal > BoundsMaxReg {
+			panic("bad yReg")
+		}
+		c |= yVal << 7
+	} else {
+		if yVal < 0 || yVal > BoundsMaxConst {
+			panic("bad yConst")
+		}
+		c |= yVal << 7
+	}
+	return c*int(numBoundsCodes) + int(code)
+}
+func BoundsDecode(v int) (code BoundsErrorCode, signed, xIsReg, yIsReg bool, xVal, yVal int) {
+	code = BoundsErrorCode(v % int(numBoundsCodes))
+	c := v / int(numBoundsCodes)
+	xIsReg = c&1 != 0
+	c >>= 1
+	yIsReg = c&1 != 0
+	c >>= 1
+	if xIsReg {
+		signed = c&1 != 0
+		c >>= 1
+		xVal = c & 0xf
+		c >>= 4
+	} else {
+		xVal = c & 0x1f
+		c >>= 5
+	}
+	if yIsReg {
+		yVal = c & 0xf
+		c >>= 4
+	} else {
+		yVal = c & 0x1f
+		c >>= 5
+	}
+	if c != 0 {
+		panic("BoundsDecode decoding error")
+	}
+	return
+}

src/internal/abi/symtab.go

@@ -79,6 +79,7 @@ const (
 	PCDATA_StackMapIndex = 1
 	PCDATA_InlTreeIndex  = 2
 	PCDATA_ArgLiveIndex  = 3
+	PCDATA_PanicBounds   = 4
 
 	FUNCDATA_ArgsPointerMaps    = 0
 	FUNCDATA_LocalsPointerMaps  = 1

src/runtime/panic.go

@@ -225,6 +225,99 @@ func panicSlice3C(x int, y int)
 func panicSlice3CU(x uint, y int)
 func panicSliceConvert(x int, y int)
 
+func panicBounds() // in asm_GOARCH.s files, called from generated code
+func panicExtend() // in asm_GOARCH.s files, called from generated code (on 32-bit archs)
+func panicBounds64(pc uintptr, regs *[16]int64) { // called from panicBounds on 64-bit archs
+	f := findfunc(pc)
+	v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1)
+
+	code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v))
+
+	if code == abi.BoundsIndex {
+		panicCheck1(pc, "index out of range")
+	} else {
+		panicCheck1(pc, "slice bounds out of range")
+	}
+
+	var e boundsError
+	e.code = code
+	e.signed = signed
+	if xIsReg {
+		e.x = regs[xVal]
+	} else {
+		e.x = int64(xVal)
+	}
+	if yIsReg {
+		e.y = int(regs[yVal])
+	} else {
+		e.y = yVal
+	}
+	panic(e)
+}
+
+func panicBounds32(pc uintptr, regs *[16]int32) { // called from panicBounds on 32-bit archs
+	f := findfunc(pc)
+	v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1)
+
+	code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v))
+
+	if code == abi.BoundsIndex {
+		panicCheck1(pc, "index out of range")
+	} else {
+		panicCheck1(pc, "slice bounds out of range")
+	}
+
+	var e boundsError
+	e.code = code
+	e.signed = signed
+	if xIsReg {
+		if signed {
+			e.x = int64(regs[xVal])
+		} else {
+			e.x = int64(uint32(regs[xVal]))
+		}
+	} else {
+		e.x = int64(xVal)
+	}
+	if yIsReg {
+		e.y = int(regs[yVal])
+	} else {
+		e.y = yVal
+	}
+	panic(e)
+}
+
+func panicBounds32X(pc uintptr, regs *[16]int32) { // called from panicExtend on 32-bit archs
+	f := findfunc(pc)
+	v := pcdatavalue(f, abi.PCDATA_PanicBounds, pc-1)
+
+	code, signed, xIsReg, yIsReg, xVal, yVal := abi.BoundsDecode(int(v))
+
+	if code == abi.BoundsIndex {
+		panicCheck1(pc, "index out of range")
+	} else {
+		panicCheck1(pc, "slice bounds out of range")
+	}
+
+	var e boundsError
+	e.code = code
+	e.signed = signed
+	if xIsReg {
+		// Our 4-bit register numbers are actually 2 2-bit register numbers.
+		lo := xVal & 3
+		hi := xVal >> 2
+		e.x = int64(regs[hi])<<32 + int64(uint32(regs[lo]))
+	} else {
+		e.x = int64(xVal)
+	}
+	if yIsReg {
+		e.y = int(regs[yVal])
+	} else {
+		e.y = yVal
+	}
+	panic(e)
+}
+
 var shiftError = error(errorString("negative shift amount"))
 
 //go:yeswritebarrierrec