cmd/compile: call generated size-specialized malloc functions directly

This change creates calls to size-specialized malloc functions instead
of calls to newObject when we know the size of the allocation at
compilation time. Most of it is a matter of calling the newObject
function (which will create calls to the size-specialized functions)
rather then the newObjectNonSpecialized function (which won't). In the
newHeapaddr, small, non-pointer case, we'll create a non specialized
newObject and transform that into the appropriate size-specialized
function when we produce the mallocgc in flushPendingHeapAllocations.

We have to update some of the rewrites in generic.rules to also apply to
the size-specialized functions when they apply to newObject.

The messiest thing is we have to adjust the offset we use to save the
memory profiler stack, because the depth of the call to profilealloc is
two frames fewer in the size-specialized malloc functions compared to
when newObject calls mallocgc. A bunch of tests have been adjusted to
account for that.

Change-Id: I6a6a6964c9037fb6719e392c4a498ed700b617d7
Reviewed-on: https://go-review.googlesource.com/c/go/+/707856
Reviewed-by: Michael Knyszek <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Keith Randall <[email protected]>

Author: matloob

src/cmd/compile/internal/ir/symtab.go

@@ -13,47 +13,50 @@ import (
 var Syms symsStruct
 
 type symsStruct struct {
-	AssertE2I         *obj.LSym
-	AssertE2I2        *obj.LSym
-	Asanread          *obj.LSym
-	Asanwrite         *obj.LSym
-	CgoCheckMemmove   *obj.LSym
-	CgoCheckPtrWrite  *obj.LSym
-	CheckPtrAlignment *obj.LSym
-	Deferproc         *obj.LSym
-	Deferprocat       *obj.LSym
-	DeferprocStack    *obj.LSym
-	Deferreturn       *obj.LSym
-	Duffcopy          *obj.LSym
-	Duffzero          *obj.LSym
-	GCWriteBarrier    [8]*obj.LSym
-	Goschedguarded    *obj.LSym
-	Growslice         *obj.LSym
-	InterfaceSwitch   *obj.LSym
-	MallocGC          *obj.LSym
-	Memmove           *obj.LSym
-	Msanread          *obj.LSym
-	Msanwrite         *obj.LSym
-	Msanmove          *obj.LSym
-	Newobject         *obj.LSym
-	Newproc           *obj.LSym
-	PanicBounds       *obj.LSym
-	PanicExtend       *obj.LSym
-	Panicdivide       *obj.LSym
-	Panicshift        *obj.LSym
-	PanicdottypeE     *obj.LSym
-	PanicdottypeI     *obj.LSym
-	Panicnildottype   *obj.LSym
-	Panicoverflow     *obj.LSym
-	Racefuncenter     *obj.LSym
-	Racefuncexit      *obj.LSym
-	Raceread          *obj.LSym
-	Racereadrange     *obj.LSym
-	Racewrite         *obj.LSym
-	Racewriterange    *obj.LSym
-	TypeAssert        *obj.LSym
-	WBZero            *obj.LSym
-	WBMove            *obj.LSym
+	AssertE2I                 *obj.LSym
+	AssertE2I2                *obj.LSym
+	Asanread                  *obj.LSym
+	Asanwrite                 *obj.LSym
+	CgoCheckMemmove           *obj.LSym
+	CgoCheckPtrWrite          *obj.LSym
+	CheckPtrAlignment         *obj.LSym
+	Deferproc                 *obj.LSym
+	Deferprocat               *obj.LSym
+	DeferprocStack            *obj.LSym
+	Deferreturn               *obj.LSym
+	Duffcopy                  *obj.LSym
+	Duffzero                  *obj.LSym
+	GCWriteBarrier            [8]*obj.LSym
+	Goschedguarded            *obj.LSym
+	Growslice                 *obj.LSym
+	InterfaceSwitch           *obj.LSym
+	MallocGC                  *obj.LSym
+	MallocGCSmallNoScan       [27]*obj.LSym
+	MallocGCSmallScanNoHeader [27]*obj.LSym
+	MallocGCTiny              [16]*obj.LSym
+	Memmove                   *obj.LSym
+	Msanread                  *obj.LSym
+	Msanwrite                 *obj.LSym
+	Msanmove                  *obj.LSym
+	Newobject                 *obj.LSym
+	Newproc                   *obj.LSym
+	PanicBounds               *obj.LSym
+	PanicExtend               *obj.LSym
+	Panicdivide               *obj.LSym
+	Panicshift                *obj.LSym
+	PanicdottypeE             *obj.LSym
+	PanicdottypeI             *obj.LSym
+	Panicnildottype           *obj.LSym
+	Panicoverflow             *obj.LSym
+	Racefuncenter             *obj.LSym
+	Racefuncexit              *obj.LSym
+	Raceread                  *obj.LSym
+	Racereadrange             *obj.LSym
+	Racewrite                 *obj.LSym
+	Racewriterange            *obj.LSym
+	TypeAssert                *obj.LSym
+	WBZero                    *obj.LSym
+	WBMove                    *obj.LSym
 	// Wasm
 	SigPanic         *obj.LSym
 	Staticuint64s    *obj.LSym

src/cmd/compile/internal/ssa/_gen/generic.rules

@@ -2065,28 +2065,32 @@
 // for rewriting results of some late-expanded rewrites (below)
 (SelectN [n] m:(MakeResult ___)) => m.Args[n]
 
+// TODO(matloob): Try out having non-zeroing mallocs for prointerless
+// memory, and leaving the zeroing here. Then the compiler can remove
+// the zeroing if the user has explicit writes to the whole object.
+
 // for late-expanded calls, recognize newobject and remove zeroing and nilchecks
-(Zero (SelectN [0] call:(StaticLECall _ _)) mem:(SelectN [1] call))
-	&& isSameCall(call.Aux, "runtime.newobject")
+(Zero (SelectN [0] call:(StaticLECall ___)) mem:(SelectN [1] call))
+	&& isMalloc(call.Aux)
 	=> mem
 
-(Store (SelectN [0] call:(StaticLECall _ _)) x mem:(SelectN [1] call))
+(Store (SelectN [0] call:(StaticLECall ___)) x mem:(SelectN [1] call))
 	&& isConstZero(x)
-	&& isSameCall(call.Aux, "runtime.newobject")
+	&& isMalloc(call.Aux)
 	=> mem
 
-(Store (OffPtr (SelectN [0] call:(StaticLECall _ _))) x mem:(SelectN [1] call))
+(Store (OffPtr (SelectN [0] call:(StaticLECall ___))) x mem:(SelectN [1] call))
 	&& isConstZero(x)
-	&& isSameCall(call.Aux, "runtime.newobject")
+	&& isMalloc(call.Aux)
 	=> mem
 
-(NilCheck ptr:(SelectN [0] call:(StaticLECall _ _)) _)
-	&& isSameCall(call.Aux, "runtime.newobject")
+(NilCheck ptr:(SelectN [0] call:(StaticLECall ___)) _)
+	&& isMalloc(call.Aux)
 	&& warnRule(fe.Debug_checknil(), v, "removed nil check")
 	=> ptr
 
-(NilCheck ptr:(OffPtr (SelectN [0] call:(StaticLECall _ _))) _)
-	&& isSameCall(call.Aux, "runtime.newobject")
+(NilCheck ptr:(OffPtr (SelectN [0] call:(StaticLECall ___))) _)
+	&& isMalloc(call.Aux)
 	&& warnRule(fe.Debug_checknil(), v, "removed nil check")
 	=> ptr
 

src/cmd/compile/internal/ssa/rewrite.go

@@ -456,6 +456,26 @@ func isSameCall(aux Aux, name string) bool {
 	return fn != nil && fn.String() == name
 }
 
+func isMalloc(aux Aux) bool {
+	return isNewObject(aux) || isSpecializedMalloc(aux)
+}
+
+func isNewObject(aux Aux) bool {
+	fn := aux.(*AuxCall).Fn
+	return fn != nil && fn.String() == "runtime.newobject"
+}
+
+func isSpecializedMalloc(aux Aux) bool {
+	fn := aux.(*AuxCall).Fn
+	if fn == nil {
+		return false
+	}
+	name := fn.String()
+	return strings.HasPrefix(name, "runtime.mallocgcSmallNoScanSC") ||
+		strings.HasPrefix(name, "runtime.mallocgcSmallScanNoHeaderSC") ||
+		strings.HasPrefix(name, "runtime.mallocTiny")
+}
+
 // canLoadUnaligned reports if the architecture supports unaligned load operations.
 func canLoadUnaligned(c *Config) bool {
 	return c.ctxt.Arch.Alignment == 1

src/cmd/compile/internal/ssa/rewritegeneric.go

@@ -798,7 +798,16 @@ func IsNewObject(v *Value, select1 []*Value) (mem *Value, ok bool) {
 	if call.Op != OpStaticCall {
 		return nil, false
 	}
-	if !isSameCall(call.Aux, "runtime.newobject") {
+	// Check for new object, or for new object calls that have been transformed into size-specialized malloc calls.
+	// Calls that have return type unsafe pointer may have originally been produced by flushPendingHeapAllocations
+	// in the ssa generator, so may have not originally been newObject calls.
+	var numParameters int64
+	switch {
+	case isNewObject(call.Aux):
+		numParameters = 1
+	case isSpecializedMalloc(call.Aux) && !v.Type.IsUnsafePtr():
+		numParameters = 3
+	default:
 		return nil, false
 	}
 	if f.ABIDefault == f.ABI1 && len(c.intParamRegs) >= 1 {
@@ -813,7 +822,7 @@ func IsNewObject(v *Value, select1 []*Value) (mem *Value, ok bool) {
 	if v.Args[0].Args[0].Op != OpSP {
 		return nil, false
 	}
-	if v.Args[0].AuxInt != c.ctxt.Arch.FixedFrameSize+c.RegSize { // offset of return value
+	if v.Args[0].AuxInt != c.ctxt.Arch.FixedFrameSize+numParameters*c.RegSize { // offset of return value
 		return nil, false
 	}
 	return mem, true