runtime: track the last allocated block to minimize heap scanning

Author: dgryski

src/runtime/gc_blocks.go

@@ -52,6 +52,7 @@ var (
 	metadataStart unsafe.Pointer // pointer to the start of the heap metadata
 	nextAlloc     gcBlock        // the next block that should be tried by the allocator
 	endBlock      gcBlock        // the block just past the end of the available space
+	maxBlock      gcBlock        // maximum block that has been allocated
 	gcTotalAlloc  uint64         // total number of bytes allocated
 	gcTotalBlocks uint64         // total number of allocated blocks
 	gcMallocs     uint64         // total number of allocations
@@ -377,6 +378,9 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 				size -= add
 			}
 			memzero(pointer, size)
+			if nextAlloc > maxBlock {
+				maxBlock = nextAlloc - 1
+			}
 			return pointer
 		}
 	}
@@ -586,7 +590,7 @@ func finishMark() {
 	for stackOverflow {
 		// Re-mark all blocks.
 		stackOverflow = false
-		for block := gcBlock(0); block < endBlock; block++ {
+		for block := gcBlock(0); block <= maxBlock; block++ {
 			if block.state() != blockStateMark {
 				// Block is not marked, so we do not need to rescan it.
 				continue
@@ -621,9 +625,10 @@ func markRoot(addr, root uintptr) {
 // Sweep goes through all memory and frees unmarked memory.
 // It returns how many bytes are free in the heap after the sweep.
 func sweep() (freeBytes uintptr) {
+	var lastBlock gcBlock
 	freeCurrentObject := false
 	var freed uint64
-	for block := gcBlock(0); block < endBlock; block++ {
+	for block := gcBlock(0); block <= maxBlock; block++ {
 		switch block.state() {
 		case blockStateHead:
 			// Unmarked head. Free it, including all tail blocks following it.
@@ -637,19 +642,29 @@ func sweep() (freeBytes uintptr) {
 				// Free it now.
 				block.markFree()
 				freed++
+				freeBytes += bytesPerBlock
+			} else {
+				lastBlock = block
 			}
 		case blockStateMark:
 			// This is a marked object. The next tail blocks must not be freed,
 			// but the mark bit must be removed so the next GC cycle will
 			// collect this object if it is unreferenced then.
 			block.unmark()
 			freeCurrentObject = false
+			lastBlock = block
 		case blockStateFree:
 			freeBytes += bytesPerBlock
 		}
 	}
+
 	gcFreedBlocks += freed
 	freeBytes += uintptr(freed) * bytesPerBlock
+
+	if gcAsserts && lastBlock > maxBlock {
+		runtimePanic("gc: lastblock > maxBlock")
+	}
+	maxBlock = lastBlock
 	return
 }