diff --git a/builder/sizes_test.go b/builder/sizes_test.go index 45c1ca74eb..b29c9d669a 100644 --- a/builder/sizes_test.go +++ b/builder/sizes_test.go @@ -42,9 +42,9 @@ func TestBinarySize(t *testing.T) { // This is a small number of very diverse targets that we want to test. tests := []sizeTest{ // microcontrollers - {"hifive1b", "examples/echo", 3668, 280, 0, 2244}, - {"microbit", "examples/serial", 2694, 342, 8, 2248}, - {"wioterminal", "examples/pininterrupt", 7187, 1489, 116, 6888}, + {"hifive1b", "examples/echo", 3684, 280, 0, 2244}, + {"microbit", "examples/serial", 2710, 342, 8, 2248}, + {"wioterminal", "examples/pininterrupt", 7207, 1489, 116, 6888}, // TODO: also check wasm. Right now this is difficult, because // wasm binaries are run through wasm-opt and therefore the diff --git a/src/internal/task/task_asyncify.go b/src/internal/task/task_asyncify.go index 546b6841b2..a08be7f733 100644 --- a/src/internal/task/task_asyncify.go +++ b/src/internal/task/task_asyncify.go @@ -58,9 +58,6 @@ func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { //export tinygo_launch func (*state) launch() -//go:linkname align runtime.align -func align(p uintptr) uintptr - // initialize the state and prepare to call the specified function with the specified argument bundle. func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) { // Save the entry call. diff --git a/src/runtime/arch_386.go b/src/runtime/arch_386.go index 90ec8e8baf..39eb02940f 100644 --- a/src/runtime/arch_386.go +++ b/src/runtime/arch_386.go @@ -16,10 +16,9 @@ const ( linux_SIGSEGV = 11 ) -// Align on word boundary. -func align(ptr uintptr) uintptr { - return (ptr + 15) &^ 15 -} +// maxAlign is the maximum alignment required from the memory allocator. +// The ABI requires 16-byte alignment for the stack and vectors. +const maxAlign = 16 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_amd64.go b/src/runtime/arch_amd64.go index 436d6e3849..213a1c9333 100644 --- a/src/runtime/arch_amd64.go +++ b/src/runtime/arch_amd64.go @@ -16,12 +16,9 @@ const ( linux_SIGSEGV = 11 ) -// Align a pointer. -// Note that some amd64 instructions (like movaps) expect 16-byte aligned -// memory, thus the result must be 16-byte aligned. -func align(ptr uintptr) uintptr { - return (ptr + 15) &^ 15 -} +// maxAlign is the maximum alignment required from the memory allocator. +// The ABI requires 16-byte alignment for the stack and vectors. +const maxAlign = 16 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_arm.go b/src/runtime/arch_arm.go index ea6b540d2a..dc47102564 100644 --- a/src/runtime/arch_arm.go +++ b/src/runtime/arch_arm.go @@ -18,10 +18,9 @@ const ( linux_SIGSEGV = 11 ) -// Align on the maximum alignment for this platform (double). -func align(ptr uintptr) uintptr { - return (ptr + 7) &^ 7 -} +// maxAlign is the maximum alignment required from the memory allocator. +// EABI requires 8-byte alignment for the stack and 64-bit values. +const maxAlign = 8 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_arm64.go b/src/runtime/arch_arm64.go index 6d3c856cf6..60090f2034 100644 --- a/src/runtime/arch_arm64.go +++ b/src/runtime/arch_arm64.go @@ -16,10 +16,9 @@ const ( linux_SIGSEGV = 11 ) -// Align on word boundary. -func align(ptr uintptr) uintptr { - return (ptr + 15) &^ 15 -} +// maxAlign is the maximum alignment required from the memory allocator. +// The ABI requires 16-byte alignment for the stack and vectors. +const maxAlign = 16 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_avr.go b/src/runtime/arch_avr.go index 251d154354..415d98c252 100644 --- a/src/runtime/arch_avr.go +++ b/src/runtime/arch_avr.go @@ -13,11 +13,9 @@ const deferExtraRegs = 1 // the frame pointer (Y register) also needs to be stor const callInstSize = 2 // "call" is 4 bytes, "rcall" is 2 bytes -// Align on a word boundary. -func align(ptr uintptr) uintptr { - // No alignment necessary on the AVR. - return ptr -} +// maxAlign is the maximum alignment required from the memory allocator. +// The ABI never requires alignment. +const maxAlign = 1 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_cortexm.go b/src/runtime/arch_cortexm.go index 6ea4a4838e..74a1bb638b 100644 --- a/src/runtime/arch_cortexm.go +++ b/src/runtime/arch_cortexm.go @@ -15,10 +15,9 @@ const deferExtraRegs = 0 const callInstSize = 4 // "bl someFunction" is 4 bytes -// Align on word boundary. -func align(ptr uintptr) uintptr { - return (ptr + 7) &^ 7 -} +// maxAlign is the maximum alignment required from the memory allocator. +// EABI requires 8-byte alignment for the stack and 64-bit values. +const maxAlign = 8 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_mips.go b/src/runtime/arch_mips.go index 5a7d05c898..cf3a8879bb 100644 --- a/src/runtime/arch_mips.go +++ b/src/runtime/arch_mips.go @@ -16,10 +16,9 @@ const ( linux_SIGSEGV = 11 ) -// It appears that MIPS has a maximum alignment of 8 bytes. -func align(ptr uintptr) uintptr { - return (ptr + 7) &^ 7 -} +// maxAlign is the maximum alignment required from the memory allocator. +// The o32 ABI requires 8-byte alignment for the stack and float64. +const maxAlign = 8 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_mipsle.go b/src/runtime/arch_mipsle.go index 498cf862b7..f650c4fa8f 100644 --- a/src/runtime/arch_mipsle.go +++ b/src/runtime/arch_mipsle.go @@ -16,10 +16,9 @@ const ( linux_SIGSEGV = 11 ) -// It appears that MIPS has a maximum alignment of 8 bytes. -func align(ptr uintptr) uintptr { - return (ptr + 7) &^ 7 -} +// maxAlign is the maximum alignment required from the memory allocator. +// The o32 ABI requires 8-byte alignment for the stack and float64. +const maxAlign = 8 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_tinygoriscv.go b/src/runtime/arch_tinygoriscv.go index 921c775a5e..5230e59a3b 100644 --- a/src/runtime/arch_tinygoriscv.go +++ b/src/runtime/arch_tinygoriscv.go @@ -8,11 +8,10 @@ const deferExtraRegs = 0 const callInstSize = 4 // 8 without relaxation, maybe 4 with relaxation -// RISC-V has a maximum alignment of 16 bytes (both for RV32 and for RV64). +// maxAlign is the maximum alignment required from the memory allocator. +// The RISC-V ABI requires 16-byte alignment for the stack and 128-bit floats. // Source: https://riscv.org/wp-content/uploads/2015/01/riscv-calling.pdf -func align(ptr uintptr) uintptr { - return (ptr + 15) &^ 15 -} +const maxAlign = 16 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/arch_tinygowasm.go b/src/runtime/arch_tinygowasm.go index 0f61fe0082..c4555648fc 100644 --- a/src/runtime/arch_tinygowasm.go +++ b/src/runtime/arch_tinygowasm.go @@ -63,12 +63,9 @@ var ( stackTop = uintptr(unsafe.Pointer(&globalsStartSymbol)) ) -func align(ptr uintptr) uintptr { - // Align to 16, which is the alignment of max_align_t: - // https://godbolt.org/z/dYqTsWrGq - const heapAlign = 16 - return (ptr + heapAlign - 1) &^ (heapAlign - 1) -} +// maxAlign is the maximum alignment required from the memory allocator. +// The ABI requires 16-byte alignment for the stack and vectors. +const maxAlign = 16 //export tinygo_getCurrentStackPointer func getCurrentStackPointer() uintptr diff --git a/src/runtime/arch_xtensa.go b/src/runtime/arch_xtensa.go index 2ff4bf9eaf..e93e095c28 100644 --- a/src/runtime/arch_xtensa.go +++ b/src/runtime/arch_xtensa.go @@ -11,10 +11,9 @@ const deferExtraRegs = 0 const callInstSize = 3 // "callx0 someFunction" (and similar) is 3 bytes -// The largest alignment according to the Xtensa ABI is 8 (long long, double). -func align(ptr uintptr) uintptr { - return (ptr + 7) &^ 7 -} +// maxAlign is the maximum alignment required from the memory allocator. +// The ABI requires 8-byte alignment for the stack and 64-bit types. +const maxAlign = 8 func getCurrentStackPointer() uintptr { return uintptr(stacksave()) diff --git a/src/runtime/gc_blocks.go b/src/runtime/gc_blocks.go index a95edbfd0d..abd0adb537 100644 --- a/src/runtime/gc_blocks.go +++ b/src/runtime/gc_blocks.go @@ -21,7 +21,7 @@ package runtime // // Metadata is stored in a special area at the end of the heap, in the area // metadataStart..heapEnd. The actual blocks are stored in -// heapStart..metadataStart. +// getHeapBase()..metadataStart. // // More information: // https://aykevl.nl/2020/09/gc-tinygo @@ -48,6 +48,18 @@ const ( blocksPerStateByte = 8 / stateBits ) +// objHeaderSize is the amount of space occupied by the objHeader at the start of an allocation. +const objHeaderSize = unsafe.Sizeof(objHeader{}) + +// reverseAlignHeap is the amount to shift the heap forwards by. +// If the object header size is not aligned, shifting the whole heap forwards will ensure that the end of each object header is properly aligned. +const reverseAlignHeap = maxAlign - objHeaderSize%maxAlign + +// getHeapBase returns the starting address of the heap after reverse-alignment. +func getHeapBase() uintptr { + return heapStart + reverseAlignHeap +} + var ( metadataStart unsafe.Pointer // pointer to the start of the heap metadata scanList *objHeader // scanList is a singly linked list of heap objects that have been marked but not scanned @@ -109,10 +121,10 @@ type gcBlock uintptr // blockFromAddr returns a block given an address somewhere in the heap (which // might not be heap-aligned). func blockFromAddr(addr uintptr) gcBlock { - if gcAsserts && (addr < heapStart || addr >= uintptr(metadataStart)) { + if gcAsserts && (addr < getHeapBase() || addr >= uintptr(metadataStart)) { runtimePanic("gc: trying to get block from invalid address") } - return gcBlock((addr - heapStart) / bytesPerBlock) + return gcBlock((addr - getHeapBase()) / bytesPerBlock) } // Return a pointer to the start of the allocated object. @@ -122,7 +134,7 @@ func (b gcBlock) pointer() unsafe.Pointer { // Return the address of the start of the allocated object. func (b gcBlock) address() uintptr { - addr := heapStart + uintptr(b)*bytesPerBlock + addr := getHeapBase() + uintptr(b)*bytesPerBlock if gcAsserts && addr > uintptr(metadataStart) { runtimePanic("gc: block pointing inside metadata") } @@ -303,7 +315,7 @@ func popFreeRange(len uintptr) unsafe.Pointer { } func isOnHeap(ptr uintptr) bool { - return ptr >= heapStart && ptr < uintptr(metadataStart) + return ptr >= getHeapBase() && ptr < uintptr(metadataStart) } // Initialize the memory allocator. @@ -356,22 +368,22 @@ func setHeapEnd(newHeapEnd uintptr) { } // calculateHeapAddresses initializes variables such as metadataStart and -// numBlock based on heapStart and heapEnd. +// numBlock based on getHeapBase() and heapEnd. // // This function can be called again when the heap size increases. The caller is // responsible for copying the metadata to the new location. func calculateHeapAddresses() { - totalSize := heapEnd - heapStart + totalSize := heapEnd - getHeapBase() // Allocate some memory to keep 2 bits of information about every block. metadataSize := (totalSize + blocksPerStateByte*bytesPerBlock) / (1 + blocksPerStateByte*bytesPerBlock) metadataStart = unsafe.Pointer(heapEnd - metadataSize) // Use the rest of the available memory as heap. - numBlocks := (uintptr(metadataStart) - heapStart) / bytesPerBlock + numBlocks := (uintptr(metadataStart) - getHeapBase()) / bytesPerBlock endBlock = gcBlock(numBlocks) if gcDebug { - println("heapStart: ", heapStart) + println("heapBase: ", getHeapBase()) println("heapEnd: ", heapEnd) println("total size: ", totalSize) println("metadata size: ", metadataSize) @@ -400,7 +412,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { // Round the size up to a multiple of blocks, adding space for the header. rawSize := size - size += align(unsafe.Sizeof(objHeader{})) + size += objHeaderSize size += bytesPerBlock - 1 if size < rawSize { // The size overflowed. @@ -431,7 +443,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { // Run the collector and try again. freeBytes := runGC() ranGC = true - heapSize := uintptr(metadataStart) - heapStart + heapSize := uintptr(metadataStart) - getHeapBase() if freeBytes < heapSize/3 { // Ensure there is at least 33% headroom. // This percentage was arbitrarily chosen, and may need to @@ -471,9 +483,8 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { gcLock.Unlock() // Return a pointer to this allocation. - add := align(unsafe.Sizeof(objHeader{})) - pointer = unsafe.Add(pointer, add) - size -= add + pointer = unsafe.Add(pointer, objHeaderSize) + size -= objHeaderSize memzero(pointer, size) return pointer } @@ -631,7 +642,7 @@ func finishMark() { // Compute the scan bounds. objAddr := uintptr(unsafe.Pointer(obj)) - start := objAddr + align(unsafe.Sizeof(objHeader{})) + start := objAddr + objHeaderSize end := blockFromAddr(objAddr).findNext().address() // Scan the object. @@ -801,9 +812,9 @@ func ReadMemStats(m *MemStats) { // Calculate the raw size of the heap. heapEnd := heapEnd - heapStart := heapStart - m.Sys = uint64(heapEnd - heapStart) - m.HeapSys = uint64(uintptr(metadataStart) - heapStart) + heapBase := getHeapBase() + m.Sys = uint64(heapEnd - heapBase) + m.HeapSys = uint64(uintptr(metadataStart) - heapBase) metadataStart := metadataStart // TODO: should GCSys include objHeaders? m.GCSys = uint64(heapEnd - uintptr(metadataStart)) diff --git a/src/runtime/gc_leaking.go b/src/runtime/gc_leaking.go index b60fff1045..a0a3d4d1fa 100644 --- a/src/runtime/gc_leaking.go +++ b/src/runtime/gc_leaking.go @@ -37,7 +37,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { // much. And by using platform-native data types (e.g. *uint8 for 8-bit // systems). gcLock.Lock() - size = align(size) + size = (size + maxAlign - 1) &^ (maxAlign - 1) addr := heapptr gcTotalAlloc += uint64(size) gcMallocs++