diff --git a/compiler/atomic.go b/compiler/atomic.go index 496e3a2c9f..8ea75037d1 100644 --- a/compiler/atomic.go +++ b/compiler/atomic.go @@ -20,7 +20,7 @@ func (b *builder) createAtomicOp(name string) llvm.Value { val := b.getValue(b.fn.Params[1], getPos(b.fn)) oldVal := b.CreateAtomicRMW(llvm.AtomicRMWBinOpAnd, ptr, val, llvm.AtomicOrderingSequentiallyConsistent, true) return oldVal - case "OrInt32", "OrInt64", "OrUint32", "OrUint64", "OrUintptr": + case "atomicOr8", "OrInt32", "OrInt64", "OrUint32", "OrUint64", "OrUintptr": ptr := b.getValue(b.fn.Params[0], getPos(b.fn)) val := b.getValue(b.fn.Params[1], getPos(b.fn)) oldVal := b.CreateAtomicRMW(llvm.AtomicRMWBinOpOr, ptr, val, llvm.AtomicOrderingSequentiallyConsistent, true) @@ -37,7 +37,7 @@ func (b *builder) createAtomicOp(name string) llvm.Value { tuple := b.CreateAtomicCmpXchg(ptr, old, newVal, llvm.AtomicOrderingSequentiallyConsistent, llvm.AtomicOrderingSequentiallyConsistent, true) swapped := b.CreateExtractValue(tuple, 1, "") return swapped - case "LoadInt32", "LoadInt64", "LoadUint32", "LoadUint64", "LoadUintptr", "LoadPointer": + case "atomicLoad8", "LoadInt32", "LoadInt64", "LoadUint32", "LoadUint64", "LoadUintptr", "LoadPointer": ptr := b.getValue(b.fn.Params[0], getPos(b.fn)) val := b.CreateLoad(b.getLLVMType(b.fn.Signature.Results().At(0).Type()), ptr, "") val.SetOrdering(llvm.AtomicOrderingSequentiallyConsistent) diff --git a/compiler/intrinsics.go b/compiler/intrinsics.go index 571009c861..7037e8ef9d 100644 --- a/compiler/intrinsics.go +++ b/compiler/intrinsics.go @@ -33,7 +33,7 @@ func (b *builder) defineIntrinsicFunction() { b.createVolatileLoad() case strings.HasPrefix(name, "runtime/volatile.Store"): b.createVolatileStore() - case strings.HasPrefix(name, "sync/atomic.") && token.IsExported(b.fn.Name()): + case (strings.HasPrefix(name, "sync/atomic.") && token.IsExported(b.fn.Name())) || strings.HasPrefix(name, "runtime.atomic"): b.createFunctionStart(true) returnValue := b.createAtomicOp(b.fn.Name()) if !returnValue.IsNil() { diff --git a/src/internal/task/task_threads.c b/src/internal/task/task_threads.c index c60a2fe7d8..5ea6164eb7 100644 --- a/src/internal/task/task_threads.c +++ b/src/internal/task/task_threads.c @@ -147,3 +147,10 @@ void* tinygo_task_current(void) { void tinygo_task_send_gc_signal(pthread_t thread) { pthread_kill(thread, taskPauseSignal); } + +// The local gc scan list. +static __thread void *gcScanList; + +void* tinygo_scan_list(void) { + return &gcScanList; +} diff --git a/src/internal/task/task_threads.go b/src/internal/task/task_threads.go index 2085fa979f..aa7f637996 100644 --- a/src/internal/task/task_threads.go +++ b/src/internal/task/task_threads.go @@ -147,9 +147,6 @@ func taskExited(t *Task) { } } -// scanWaitGroup is used to wait on until all threads have finished the current state transition. -var scanWaitGroup waitGroup - type waitGroup struct { f Futex } @@ -176,109 +173,9 @@ func (wg *waitGroup) wait() { } } -// gcState is used to track and notify threads when the GC is stopping/resuming. -var gcState Futex - -const ( - gcStateResumed = iota - gcStateStopped -) - -// GC scan phase. Because we need to stop the world while scanning, this kinda -// needs to be done in the tasks package. -// -// After calling this function, GCResumeWorld needs to be called once to resume -// all threads again. -func GCStopWorldAndScan() { - current := Current() - - // NOTE: This does not need to be atomic. - if gcState.Load() == gcStateResumed { - // Don't allow new goroutines to be started while pausing/resuming threads - // in the stop-the-world phase. - activeTaskLock.Lock() - - // Wait for threads to finish resuming. - scanWaitGroup.wait() - - // Change the gc state to stopped. - // NOTE: This does not need to be atomic. - gcState.Store(gcStateStopped) - - // Set the number of threads to wait for. - scanWaitGroup = initWaitGroup(otherGoroutines) - - // Pause all other threads. - for t := activeTasks; t != nil; t = t.state.QueueNext { - if t != current { - tinygo_task_send_gc_signal(t.state.thread) - } - } - - // Wait for the threads to finish stopping. - scanWaitGroup.wait() - } - - // Scan other thread stacks. - for t := activeTasks; t != nil; t = t.state.QueueNext { - if t != current { - markRoots(t.state.stackBottom, t.state.stackTop) - } - } - - // Scan the current stack, and all current registers. - scanCurrentStack() - - // Scan all globals (implemented in the runtime). - gcScanGlobals() -} - -// After the GC is done scanning, resume all other threads. -func GCResumeWorld() { - // NOTE: This does not need to be atomic. - if gcState.Load() == gcStateResumed { - // This is already resumed. - return - } - - // Set the wait group to track resume progress. - scanWaitGroup = initWaitGroup(otherGoroutines) - - // Set the state to resumed. - gcState.Store(gcStateResumed) - - // Wake all of the stopped threads. - gcState.WakeAll() - - // Allow goroutines to start and exit again. - activeTaskLock.Unlock() -} - -//go:linkname markRoots runtime.markRoots -func markRoots(start, end uintptr) - // Scan globals, implemented in the runtime package. func gcScanGlobals() -var stackScanLock PMutex - -//export tinygo_task_gc_pause -func tingyo_task_gc_pause(sig int32) { - // Write the entrty stack pointer to the state. - Current().state.stackBottom = uintptr(stacksave()) - - // Notify the GC that we are stopped. - scanWaitGroup.done() - - // Wait for the GC to resume. - for gcState.Load() == gcStateStopped { - gcState.Wait(gcStateStopped) - } - - // Notify the GC that we have resumed. - scanWaitGroup.done() -} - //go:export tinygo_scanCurrentStack func scanCurrentStack() diff --git a/src/internal/task/task_threads_blocks.go b/src/internal/task/task_threads_blocks.go new file mode 100644 index 0000000000..cda1dcaf28 --- /dev/null +++ b/src/internal/task/task_threads_blocks.go @@ -0,0 +1,88 @@ +//go:build scheduler.threads && (gc.conservative || gc.precise) + +package task + +// stopWaitGroup is used to wait until all threads have stopped. +var stopWaitGroup waitGroup + +// scanWaitGroup is used to wait until all threads have finished scanning their stacks. +var scanWaitGroup waitGroup + +// resumeWaitGroup is used to wait until all threads have resumed. +var resumeWaitGroup waitGroup + +// GC scan phase. Because we need to stop the world while scanning, this kinda +// needs to be done in the tasks package. +// +// After calling this function, GCResumeWorld needs to be called once to resume +// all threads again. +func GCStopWorldAndScan() { + // Wait for threads to resume from the previous scan. + resumeWaitGroup.wait() + + // Don't allow new goroutines to be started while pausing/resuming threads + // in the stop-the-world phase. + activeTaskLock.Lock() + + // Set the number of threads to wait for. + otherGoroutines := otherGoroutines + stopWaitGroup = initWaitGroup(otherGoroutines) + scanWaitGroup = initWaitGroup(otherGoroutines + 1) + resumeWaitGroup = initWaitGroup(otherGoroutines) + + // Pause all other threads. + current := Current() + for t := activeTasks; t != nil; t = t.state.QueueNext { + if t != current { + tinygo_task_send_gc_signal(t.state.thread) + } + } + + // Wait for everything to stop. + stopWaitGroup.wait() + + // Scan all globals (implemented in the runtime). + gcScanGlobals() + + // Scan our stack and wait for everything else to complete. + localScan() +} + +//export tinygo_task_gc_pause +func tingyo_task_gc_pause(sig int32) { + // We have stopped. + stopWaitGroup.done() + + // Wait for all other threads to stop. + stopWaitGroup.wait() + + // Scan the local stack. + localScan() + + // We are resuming. + resumeWaitGroup.done() +} + +func localScan() { + // Scan the current stack, and all current registers. + scanCurrentStack() + + // Assist scanning of heap objects. + finishMark() + + // We are done scanning. + scanWaitGroup.done() + + // Wait for all other threads to finish scanning. + scanWaitGroup.wait() +} + +//go:linkname finishMark runtime.finishMark +func finishMark() + +// GCResumeWorld does not resume anything with the blocks collector. +// The threads will resume as soon as the scan completes. +func GCResumeWorld() { + // Allow goroutines to start and exit again. + activeTaskLock.Unlock() +} diff --git a/src/internal/task/task_threads_other.go b/src/internal/task/task_threads_other.go new file mode 100644 index 0000000000..e3fdf8bc29 --- /dev/null +++ b/src/internal/task/task_threads_other.go @@ -0,0 +1,104 @@ +//go:build scheduler.threads && !(gc.conservative || gc.precise) + +package task + +// scanWaitGroup is used to wait on until all threads have finished the current state transition. +var scanWaitGroup waitGroup + +// gcState is used to track and notify threads when the GC is stopping/resuming. +var gcState Futex + +const ( + gcStateResumed = iota + gcStateStopped +) + +// GC scan phase. Because we need to stop the world while scanning, this kinda +// needs to be done in the tasks package. +// +// After calling this function, GCResumeWorld needs to be called once to resume +// all threads again. +func GCStopWorldAndScan() { + current := Current() + + // NOTE: This does not need to be atomic. + if gcState.Load() == gcStateResumed { + // Don't allow new goroutines to be started while pausing/resuming threads + // in the stop-the-world phase. + activeTaskLock.Lock() + + // Wait for threads to finish resuming. + scanWaitGroup.wait() + + // Change the gc state to stopped. + // NOTE: This does not need to be atomic. + gcState.Store(gcStateStopped) + + // Set the number of threads to wait for. + scanWaitGroup = initWaitGroup(otherGoroutines) + + // Pause all other threads. + for t := activeTasks; t != nil; t = t.state.QueueNext { + if t != current { + tinygo_task_send_gc_signal(t.state.thread) + } + } + + // Wait for the threads to finish stopping. + scanWaitGroup.wait() + } + + // Scan other thread stacks. + for t := activeTasks; t != nil; t = t.state.QueueNext { + if t != current { + markRoots(t.state.stackBottom, t.state.stackTop) + } + } + + // Scan the current stack, and all current registers. + scanCurrentStack() + + // Scan all globals (implemented in the runtime). + gcScanGlobals() +} + +// After the GC is done scanning, resume all other threads. +func GCResumeWorld() { + // NOTE: This does not need to be atomic. + if gcState.Load() == gcStateResumed { + // This is already resumed. + return + } + + // Set the wait group to track resume progress. + scanWaitGroup = initWaitGroup(otherGoroutines) + + // Set the state to resumed. + gcState.Store(gcStateResumed) + + // Wake all of the stopped threads. + gcState.WakeAll() + + // Allow goroutines to start and exit again. + activeTaskLock.Unlock() +} + +//go:linkname markRoots runtime.markRoots +func markRoots(start, end uintptr) + +//export tinygo_task_gc_pause +func tingyo_task_gc_pause(sig int32) { + // Write the entrty stack pointer to the state. + Current().state.stackBottom = uintptr(stacksave()) + + // Notify the GC that we are stopped. + scanWaitGroup.done() + + // Wait for the GC to resume. + for gcState.Load() == gcStateStopped { + gcState.Wait(gcStateStopped) + } + + // Notify the GC that we have resumed. + scanWaitGroup.done() +} diff --git a/src/runtime/gc_blocks.go b/src/runtime/gc_blocks.go index 99ad6a8591..1918d8af11 100644 --- a/src/runtime/gc_blocks.go +++ b/src/runtime/gc_blocks.go @@ -50,7 +50,6 @@ const ( var ( metadataStart unsafe.Pointer // pointer to the start of the heap metadata - scanList *objHeader // scanList is a singly linked list of heap objects that have been marked but not scanned nextAlloc gcBlock // the next block that should be tried by the allocator endBlock gcBlock // the block just past the end of the available space gcTotalAlloc uint64 // total number of bytes allocated @@ -139,7 +138,7 @@ func (b gcBlock) findHead() gcBlock { // state byte. // This optimization speeds up findHead for pointers that point into a // large allocation. - stateByte := b.stateByte() + stateByte := b.stateByteAtomic() if stateByte == blockStateByteAllTails { b -= (b % blocksPerStateByte) + 1 continue @@ -154,7 +153,9 @@ func (b gcBlock) findHead() gcBlock { b-- } if gcAsserts { - if b.state() != blockStateHead && b.state() != blockStateMark { + switch b.stateAtomic() { + case blockStateHead, blockStateMark: + default: runtimePanic("gc: found tail without head") } } @@ -164,10 +165,11 @@ func (b gcBlock) findHead() gcBlock { // findNext returns the first block just past the end of the tail. This may or // may not be the head of an object. func (b gcBlock) findNext() gcBlock { - if b.state() == blockStateHead || b.state() == blockStateMark { + switch b.stateAtomic() { + case blockStateHead, blockStateMark: b++ } - for b.address() < uintptr(metadataStart) && b.state() == blockStateTail { + for b.address() < uintptr(metadataStart) && b.stateAtomic() == blockStateTail { b++ } return b @@ -572,19 +574,12 @@ func markCurrentGoroutineStack(sp uintptr) { func finishMark() { for { // Remove an object from the scan list. - obj := scanList + scanList := getScanList() + obj := *scanList if obj == nil { return } - scanList = obj.next - - // Check if the object may contain pointers. - if obj.layout.pointerFree() { - // This object doesn't contain any pointers. - // This is a fast path for objects like make([]int, 4096). - // It skips the length calculation. - continue - } + *scanList = obj.next // Compute the scan bounds. objAddr := uintptr(unsafe.Pointer(obj)) @@ -615,19 +610,26 @@ func markRoot(addr, root uintptr) { head := block.findHead() // Mark the object. - if head.state() == blockStateMark { + if !head.mark() { // This object is already marked. return } if gcDebug { println("found unmarked pointer", root, "at address", addr) } - head.setState(blockStateMark) + + // Check if the object may contain pointers. + obj := (*objHeader)(head.pointer()) + if obj.layout.pointerFree() { + // This object doesn't contain any pointers. + // This is a fast path for objects like make([]int, 4096). + return + } // Add the object to the scan list. - header := (*objHeader)(head.pointer()) - header.next = scanList - scanList = header + scanList := getScanList() + obj.next = *scanList + *scanList = obj } // Sweep goes through all memory and frees unmarked memory. diff --git a/src/runtime/gc_blocks_parallel.go b/src/runtime/gc_blocks_parallel.go new file mode 100644 index 0000000000..d2203ea5a7 --- /dev/null +++ b/src/runtime/gc_blocks_parallel.go @@ -0,0 +1,25 @@ +//go:build (gc.conservative || gc.precise) && scheduler.threads + +package runtime + +import "unsafe" + +//export tinygo_scan_list +func getScanList() **objHeader + +func (b gcBlock) stateAtomic() blockState { + return b.stateFromByte(b.stateByteAtomic()) +} + +func (b gcBlock) stateByteAtomic() byte { + return atomicLoad8((*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte))) +} + +func atomicLoad8(ptr *uint8) uint8 + +func (b gcBlock) mark() bool { + mask := byte(blockStateMark) << ((b % blocksPerStateByte) * stateBits) + return mask&^atomicOr8((*uint8)(unsafe.Add(metadataStart, b/blocksPerStateByte)), mask) != 0 +} + +func atomicOr8(ptr *uint8, mask uint8) uint8 diff --git a/src/runtime/gc_blocks_singlethread.go b/src/runtime/gc_blocks_singlethread.go new file mode 100644 index 0000000000..135c8ff5ec --- /dev/null +++ b/src/runtime/gc_blocks_singlethread.go @@ -0,0 +1,26 @@ +//go:build (gc.conservative || gc.precise) && !scheduler.threads + +package runtime + +// scanList is a singly linked list of heap objects that have been marked but not scanned. +var scanList *objHeader + +func getScanList() **objHeader { + return &scanList +} + +func (b gcBlock) stateAtomic() blockState { + return b.state() +} + +func (b gcBlock) stateByteAtomic() byte { + return b.stateByte() +} + +func (b gcBlock) mark() bool { + if b.state() == blockStateMark { + return false + } + b.setState(blockStateMark) + return true +}