runtime (gc_blocks.go): reverse-align the heap start

builder/sizes_test.go

@@ -42,9 +42,9 @@ func TestBinarySize(t *testing.T) {
 	// This is a small number of very diverse targets that we want to test.
 	tests := []sizeTest{
 		// microcontrollers
-		{"hifive1b", "examples/echo", 3668, 280, 0, 2244},
-		{"microbit", "examples/serial", 2694, 342, 8, 2248},
-		{"wioterminal", "examples/pininterrupt", 7187, 1489, 116, 6888},
+		{"hifive1b", "examples/echo", 3684, 280, 0, 2244},
+		{"microbit", "examples/serial", 2710, 342, 8, 2248},
+		{"wioterminal", "examples/pininterrupt", 7207, 1489, 116, 6888},
 
 		// TODO: also check wasm. Right now this is difficult, because
 		// wasm binaries are run through wasm-opt and therefore the

src/internal/task/task_asyncify.go

@@ -58,9 +58,6 @@ func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
 //export tinygo_launch
 func (*state) launch()
 
-//go:linkname align runtime.align
-func align(p uintptr) uintptr
-
 // initialize the state and prepare to call the specified function with the specified argument bundle.
 func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
 	// Save the entry call.

src/runtime/arch_386.go

@@ -16,10 +16,9 @@ const (
 	linux_SIGSEGV       = 11
 )
 
-// Align on word boundary.
-func align(ptr uintptr) uintptr {
-	return (ptr + 15) &^ 15
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The ABI requires 16-byte alignment for the stack and vectors.
+const maxAlign = 16
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_amd64.go

@@ -16,12 +16,9 @@ const (
 	linux_SIGSEGV       = 11
 )
 
-// Align a pointer.
-// Note that some amd64 instructions (like movaps) expect 16-byte aligned
-// memory, thus the result must be 16-byte aligned.
-func align(ptr uintptr) uintptr {
-	return (ptr + 15) &^ 15
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The ABI requires 16-byte alignment for the stack and vectors.
+const maxAlign = 16
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_arm.go

@@ -18,10 +18,9 @@ const (
 	linux_SIGSEGV       = 11
 )
 
-// Align on the maximum alignment for this platform (double).
-func align(ptr uintptr) uintptr {
-	return (ptr + 7) &^ 7
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// EABI requires 8-byte alignment for the stack and 64-bit values.
+const maxAlign = 8
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_arm64.go

@@ -16,10 +16,9 @@ const (
 	linux_SIGSEGV       = 11
 )
 
-// Align on word boundary.
-func align(ptr uintptr) uintptr {
-	return (ptr + 15) &^ 15
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The ABI requires 16-byte alignment for the stack and vectors.
+const maxAlign = 16
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_avr.go

@@ -13,11 +13,9 @@ const deferExtraRegs = 1 // the frame pointer (Y register) also needs to be stor
 
 const callInstSize = 2 // "call" is 4 bytes, "rcall" is 2 bytes
 
-// Align on a word boundary.
-func align(ptr uintptr) uintptr {
-	// No alignment necessary on the AVR.
-	return ptr
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The ABI never requires alignment.
+const maxAlign = 1
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_cortexm.go

@@ -15,10 +15,9 @@ const deferExtraRegs = 0
 
 const callInstSize = 4 // "bl someFunction" is 4 bytes
 
-// Align on word boundary.
-func align(ptr uintptr) uintptr {
-	return (ptr + 7) &^ 7
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// EABI requires 8-byte alignment for the stack and 64-bit values.
+const maxAlign = 8
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_mips.go

@@ -16,10 +16,9 @@ const (
 	linux_SIGSEGV       = 11
 )
 
-// It appears that MIPS has a maximum alignment of 8 bytes.
-func align(ptr uintptr) uintptr {
-	return (ptr + 7) &^ 7
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The o32 ABI requires 8-byte alignment for the stack and float64.
+const maxAlign = 8
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_mipsle.go

@@ -16,10 +16,9 @@ const (
 	linux_SIGSEGV       = 11
 )
 
-// It appears that MIPS has a maximum alignment of 8 bytes.
-func align(ptr uintptr) uintptr {
-	return (ptr + 7) &^ 7
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The o32 ABI requires 8-byte alignment for the stack and float64.
+const maxAlign = 8
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_tinygoriscv.go

@@ -8,11 +8,10 @@ const deferExtraRegs = 0
 
 const callInstSize = 4 // 8 without relaxation, maybe 4 with relaxation
 
-// RISC-V has a maximum alignment of 16 bytes (both for RV32 and for RV64).
+// maxAlign is the maximum alignment required from the memory allocator.
+// The RISC-V ABI requires 16-byte alignment for the stack and 128-bit floats.
 // Source: https://riscv.org/wp-content/uploads/2015/01/riscv-calling.pdf
-func align(ptr uintptr) uintptr {
-	return (ptr + 15) &^ 15
-}
+const maxAlign = 16
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/arch_tinygowasm.go

@@ -63,12 +63,9 @@ var (
 	stackTop = uintptr(unsafe.Pointer(&globalsStartSymbol))
 )
 
-func align(ptr uintptr) uintptr {
-	// Align to 16, which is the alignment of max_align_t:
-	// https://godbolt.org/z/dYqTsWrGq
-	const heapAlign = 16
-	return (ptr + heapAlign - 1) &^ (heapAlign - 1)
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The ABI requires 16-byte alignment for the stack and vectors.
+const maxAlign = 16
 
 //export tinygo_getCurrentStackPointer
 func getCurrentStackPointer() uintptr

src/runtime/arch_xtensa.go

@@ -11,10 +11,9 @@ const deferExtraRegs = 0
 
 const callInstSize = 3 // "callx0 someFunction" (and similar) is 3 bytes
 
-// The largest alignment according to the Xtensa ABI is 8 (long long, double).
-func align(ptr uintptr) uintptr {
-	return (ptr + 7) &^ 7
-}
+// maxAlign is the maximum alignment required from the memory allocator.
+// The ABI requires 8-byte alignment for the stack and 64-bit types.
+const maxAlign = 8
 
 func getCurrentStackPointer() uintptr {
 	return uintptr(stacksave())

src/runtime/gc_blocks.go

@@ -21,7 +21,7 @@
 //
 // Metadata is stored in a special area at the end of the heap, in the area
 // metadataStart..heapEnd. The actual blocks are stored in
-// heapStart..metadataStart.
+// getHeapBase()..metadataStart.
 //
 // More information:
 // https://aykevl.nl/2020/09/gc-tinygo
@@ -48,6 +48,18 @@
 	blocksPerStateByte = 8 / stateBits
 )
 
+// objHeaderSize is the amount of space occupied by the objHeader at the start of an allocation.
+const objHeaderSize = unsafe.Sizeof(objHeader{})
+
+// reverseAlignHeap is the amount to shift the heap forwards by.
+// If the object header size is not aligned, shifting the whole heap forwards will ensure that the end of each object header is properly aligned.
+const reverseAlignHeap = maxAlign - objHeaderSize%maxAlign
+
+// getHeapBase returns the starting address of the heap after reverse-alignment.
+func getHeapBase() uintptr {
+	return heapStart + reverseAlignHeap
+}
+
 var (
 	metadataStart unsafe.Pointer // pointer to the start of the heap metadata
 	scanList      *objHeader     // scanList is a singly linked list of heap objects that have been marked but not scanned
@@ -109,10 +121,10 @@
 // blockFromAddr returns a block given an address somewhere in the heap (which
 // might not be heap-aligned).
 func blockFromAddr(addr uintptr) gcBlock {
-	if gcAsserts && (addr < heapStart || addr >= uintptr(metadataStart)) {
+	if gcAsserts && (addr < getHeapBase() || addr >= uintptr(metadataStart)) {
 		runtimePanic("gc: trying to get block from invalid address")
 	}
-	return gcBlock((addr - heapStart) / bytesPerBlock)
+	return gcBlock((addr - getHeapBase()) / bytesPerBlock)
 }
 
 // Return a pointer to the start of the allocated object.
@@ -122,7 +134,7 @@
 
 // Return the address of the start of the allocated object.
 func (b gcBlock) address() uintptr {
-	addr := heapStart + uintptr(b)*bytesPerBlock
+	addr := getHeapBase() + uintptr(b)*bytesPerBlock
 	if gcAsserts && addr > uintptr(metadataStart) {
 		runtimePanic("gc: block pointing inside metadata")
 	}
@@ -303,7 +315,7 @@
 }
 
 func isOnHeap(ptr uintptr) bool {
-	return ptr >= heapStart && ptr < uintptr(metadataStart)
+	return ptr >= getHeapBase() && ptr < uintptr(metadataStart)
 }
 
 // Initialize the memory allocator.
@@ -356,22 +368,22 @@
 }
 
 // calculateHeapAddresses initializes variables such as metadataStart and
-// numBlock based on heapStart and heapEnd.
+// numBlock based on getHeapBase() and heapEnd.
 //
 // This function can be called again when the heap size increases. The caller is
 // responsible for copying the metadata to the new location.
 func calculateHeapAddresses() {
-	totalSize := heapEnd - heapStart
+	totalSize := heapEnd - getHeapBase()
 
 	// Allocate some memory to keep 2 bits of information about every block.
 	metadataSize := (totalSize + blocksPerStateByte*bytesPerBlock) / (1 + blocksPerStateByte*bytesPerBlock)
 	metadataStart = unsafe.Pointer(heapEnd - metadataSize)
 
 	// Use the rest of the available memory as heap.
-	numBlocks := (uintptr(metadataStart) - heapStart) / bytesPerBlock
+	numBlocks := (uintptr(metadataStart) - getHeapBase()) / bytesPerBlock
 	endBlock = gcBlock(numBlocks)
 	if gcDebug {
-		println("heapStart:        ", heapStart)
+		println("heapBase:        ", getHeapBase())
 		println("heapEnd:          ", heapEnd)
 		println("total size:       ", totalSize)
 		println("metadata size:    ", metadataSize)
@@ -400,7 +412,7 @@
 
 	// Round the size up to a multiple of blocks, adding space for the header.
 	rawSize := size
-	size += align(unsafe.Sizeof(objHeader{}))
+	size += objHeaderSize
 	size += bytesPerBlock - 1
 	if size < rawSize {
 		// The size overflowed.
@@ -431,7 +443,7 @@
 			// Run the collector and try again.
 			freeBytes := runGC()
 			ranGC = true
-			heapSize := uintptr(metadataStart) - heapStart
+			heapSize := uintptr(metadataStart) - getHeapBase()
 			if freeBytes < heapSize/3 {
 				// Ensure there is at least 33% headroom.
 				// This percentage was arbitrarily chosen, and may need to
@@ -471,9 +483,8 @@
 	gcLock.Unlock()
 
 	// Return a pointer to this allocation.
-	add := align(unsafe.Sizeof(objHeader{}))
-	pointer = unsafe.Add(pointer, add)
-	size -= add
+	pointer = unsafe.Add(pointer, objHeaderSize)
+	size -= objHeaderSize
 	memzero(pointer, size)
 	return pointer
 }
@@ -631,7 +642,7 @@
 
 		// Compute the scan bounds.
 		objAddr := uintptr(unsafe.Pointer(obj))
-		start := objAddr + align(unsafe.Sizeof(objHeader{}))
+		start := objAddr + objHeaderSize
 		end := blockFromAddr(objAddr).findNext().address()
 
 		// Scan the object.
@@ -801,9 +812,9 @@
 
 	// Calculate the raw size of the heap.
 	heapEnd := heapEnd
-	heapStart := heapStart
-	m.Sys = uint64(heapEnd - heapStart)
-	m.HeapSys = uint64(uintptr(metadataStart) - heapStart)
+	heapBase := getHeapBase()
+	m.Sys = uint64(heapEnd - heapBase)
+	m.HeapSys = uint64(uintptr(metadataStart) - heapBase)
 	metadataStart := metadataStart
 	// TODO: should GCSys include objHeaders?
 	m.GCSys = uint64(heapEnd - uintptr(metadataStart))

src/runtime/gc_leaking.go

@@ -37,7 +37,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 	// much. And by using platform-native data types (e.g. *uint8 for 8-bit
 	// systems).
 	gcLock.Lock()
-	size = align(size)
+	size = (size + maxAlign - 1) &^ (maxAlign - 1)
 	addr := heapptr
 	gcTotalAlloc += uint64(size)
 	gcMallocs++