runtime: eliminate global span queue [green tea]

This change removes the locked global span queue and replaces the
fixed-size local span queue with a variable-sized local span queue. The
variable-sized local span queue grows as needed to accomodate local
work. With no global span queue either, GC workers balance work amongst
themselves by stealing from each other.

The new variable-sized local span queues are inspired by the P-local
deque underlying sync.Pool. Unlike the sync.Pool deque, however, both
the owning P and stealing Ps take spans from the tail, making this
incarnation a strict queue, not a deque. This is intentional, since we
want a queue-like order to encourage objects to accumulate on each span.

These variable-sized local span queues are crucial to mark termination,
just like the global span queue was. To avoid hitting the ragged barrier
too often, we must check whether any Ps have any spans on their
variable-sized local span queues. We maintain a per-P atomic bitmask
(another pMask) that contains this state. We can also use this to speed
up stealing by skipping Ps that don't have any local spans.

The variable-sized local span queues are slower than the old fixed-size
local span queues because of the additional indirection, so this change
adds a non-atomic local fixed-size queue. This risks getting work stuck
on it, so, similarly to how workbufs work, each worker will occasionally
dump some spans onto its local variable-sized queue. This scales much
more nicely than dumping to a global queue, but is still visible to all
other Ps.

For golang#73581.

Change-Id: I814f54d9c3cc7fa7896167746e9823f50943ac22
Reviewed-on: https://go-review.googlesource.com/c/go/+/700496
Reviewed-by: Michael Pratt <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: mknyszek

src/runtime/export_test.go

@@ -1289,30 +1289,6 @@ func MSpanCountAlloc(ms *MSpan, bits []byte) int {
 	return result
 }
 
-type MSpanQueue mSpanQueue
-
-func (q *MSpanQueue) Size() int {
-	return (*mSpanQueue)(q).n
-}
-
-func (q *MSpanQueue) Push(s *MSpan) {
-	(*mSpanQueue)(q).push((*mspan)(s))
-}
-
-func (q *MSpanQueue) Pop() *MSpan {
-	s := (*mSpanQueue)(q).pop()
-	return (*MSpan)(s)
-}
-
-func (q *MSpanQueue) TakeAll(p *MSpanQueue) {
-	(*mSpanQueue)(q).takeAll((*mSpanQueue)(p))
-}
-
-func (q *MSpanQueue) PopN(n int) MSpanQueue {
-	p := (*mSpanQueue)(q).popN(n)
-	return (MSpanQueue)(p)
-}
-
 const (
 	TimeHistSubBucketBits = timeHistSubBucketBits
 	TimeHistNumSubBuckets = timeHistNumSubBuckets

src/runtime/gc_test.go

@@ -881,199 +881,6 @@ func TestWeakToStrongMarkTermination(t *testing.T) {
 	}
 }
 
-func TestMSpanQueue(t *testing.T) {
-	expectSize := func(t *testing.T, q *runtime.MSpanQueue, want int) {
-		t.Helper()
-		if got := q.Size(); got != want {
-			t.Errorf("expected size %d, got %d", want, got)
-		}
-	}
-	expectMSpan := func(t *testing.T, got, want *runtime.MSpan, op string) {
-		t.Helper()
-		if got != want {
-			t.Errorf("expected mspan %p from %s, got %p", want, op, got)
-		}
-	}
-	makeSpans := func(t *testing.T, n int) ([]*runtime.MSpan, func()) {
-		t.Helper()
-		spans := make([]*runtime.MSpan, 0, n)
-		for range cap(spans) {
-			spans = append(spans, runtime.AllocMSpan())
-		}
-		return spans, func() {
-			for i, s := range spans {
-				runtime.FreeMSpan(s)
-				spans[i] = nil
-			}
-		}
-	}
-	t.Run("Empty", func(t *testing.T) {
-		var q runtime.MSpanQueue
-		expectSize(t, &q, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-	})
-	t.Run("PushPop", func(t *testing.T) {
-		s := runtime.AllocMSpan()
-		defer runtime.FreeMSpan(s)
-
-		var q runtime.MSpanQueue
-		q.Push(s)
-		expectSize(t, &q, 1)
-		expectMSpan(t, q.Pop(), s, "pop")
-		expectMSpan(t, q.Pop(), nil, "pop")
-	})
-	t.Run("PushPopPushPop", func(t *testing.T) {
-		s0 := runtime.AllocMSpan()
-		defer runtime.FreeMSpan(s0)
-		s1 := runtime.AllocMSpan()
-		defer runtime.FreeMSpan(s1)
-
-		var q runtime.MSpanQueue
-
-		// Push and pop s0.
-		q.Push(s0)
-		expectSize(t, &q, 1)
-		expectMSpan(t, q.Pop(), s0, "pop")
-		expectMSpan(t, q.Pop(), nil, "pop")
-
-		// Push and pop s1.
-		q.Push(s1)
-		expectSize(t, &q, 1)
-		expectMSpan(t, q.Pop(), s1, "pop")
-		expectMSpan(t, q.Pop(), nil, "pop")
-	})
-	t.Run("PushPushPopPop", func(t *testing.T) {
-		s0 := runtime.AllocMSpan()
-		defer runtime.FreeMSpan(s0)
-		s1 := runtime.AllocMSpan()
-		defer runtime.FreeMSpan(s1)
-
-		var q runtime.MSpanQueue
-		q.Push(s0)
-		expectSize(t, &q, 1)
-		q.Push(s1)
-		expectSize(t, &q, 2)
-		expectMSpan(t, q.Pop(), s0, "pop")
-		expectMSpan(t, q.Pop(), s1, "pop")
-		expectMSpan(t, q.Pop(), nil, "pop")
-	})
-	t.Run("EmptyTakeAll", func(t *testing.T) {
-		var q runtime.MSpanQueue
-		var p runtime.MSpanQueue
-		expectSize(t, &p, 0)
-		expectSize(t, &q, 0)
-		p.TakeAll(&q)
-		expectSize(t, &p, 0)
-		expectSize(t, &q, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-		expectMSpan(t, p.Pop(), nil, "pop")
-	})
-	t.Run("Push4TakeAll", func(t *testing.T) {
-		spans, free := makeSpans(t, 4)
-		defer free()
-
-		var q runtime.MSpanQueue
-		for i, s := range spans {
-			expectSize(t, &q, i)
-			q.Push(s)
-			expectSize(t, &q, i+1)
-		}
-
-		var p runtime.MSpanQueue
-		p.TakeAll(&q)
-		expectSize(t, &p, 4)
-		for i := range p.Size() {
-			expectMSpan(t, p.Pop(), spans[i], "pop")
-		}
-		expectSize(t, &p, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-		expectMSpan(t, p.Pop(), nil, "pop")
-	})
-	t.Run("Push4Pop3", func(t *testing.T) {
-		spans, free := makeSpans(t, 4)
-		defer free()
-
-		var q runtime.MSpanQueue
-		for i, s := range spans {
-			expectSize(t, &q, i)
-			q.Push(s)
-			expectSize(t, &q, i+1)
-		}
-		p := q.PopN(3)
-		expectSize(t, &p, 3)
-		expectSize(t, &q, 1)
-		for i := range p.Size() {
-			expectMSpan(t, p.Pop(), spans[i], "pop")
-		}
-		expectMSpan(t, q.Pop(), spans[len(spans)-1], "pop")
-		expectSize(t, &p, 0)
-		expectSize(t, &q, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-		expectMSpan(t, p.Pop(), nil, "pop")
-	})
-	t.Run("Push4Pop0", func(t *testing.T) {
-		spans, free := makeSpans(t, 4)
-		defer free()
-
-		var q runtime.MSpanQueue
-		for i, s := range spans {
-			expectSize(t, &q, i)
-			q.Push(s)
-			expectSize(t, &q, i+1)
-		}
-		p := q.PopN(0)
-		expectSize(t, &p, 0)
-		expectSize(t, &q, 4)
-		for i := range q.Size() {
-			expectMSpan(t, q.Pop(), spans[i], "pop")
-		}
-		expectSize(t, &p, 0)
-		expectSize(t, &q, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-		expectMSpan(t, p.Pop(), nil, "pop")
-	})
-	t.Run("Push4Pop4", func(t *testing.T) {
-		spans, free := makeSpans(t, 4)
-		defer free()
-
-		var q runtime.MSpanQueue
-		for i, s := range spans {
-			expectSize(t, &q, i)
-			q.Push(s)
-			expectSize(t, &q, i+1)
-		}
-		p := q.PopN(4)
-		expectSize(t, &p, 4)
-		expectSize(t, &q, 0)
-		for i := range p.Size() {
-			expectMSpan(t, p.Pop(), spans[i], "pop")
-		}
-		expectSize(t, &p, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-		expectMSpan(t, p.Pop(), nil, "pop")
-	})
-	t.Run("Push4Pop5", func(t *testing.T) {
-		spans, free := makeSpans(t, 4)
-		defer free()
-
-		var q runtime.MSpanQueue
-		for i, s := range spans {
-			expectSize(t, &q, i)
-			q.Push(s)
-			expectSize(t, &q, i+1)
-		}
-		p := q.PopN(5)
-		expectSize(t, &p, 4)
-		expectSize(t, &q, 0)
-		for i := range p.Size() {
-			expectMSpan(t, p.Pop(), spans[i], "pop")
-		}
-		expectSize(t, &p, 0)
-		expectMSpan(t, q.Pop(), nil, "pop")
-		expectMSpan(t, p.Pop(), nil, "pop")
-	})
-}
-
 func TestDetectFinalizerAndCleanupLeaks(t *testing.T) {
 	got := runTestProg(t, "testprog", "DetectFinalizerAndCleanupLeaks", "GODEBUG=checkfinalizers=1")
 	sp := strings.SplitN(got, "detected possible issues with cleanups and/or finalizers", 2)

src/runtime/mgc.go

@@ -326,7 +326,7 @@ type workType struct {
 	full  lfstack          // lock-free list of full blocks workbuf
 	_     cpu.CacheLinePad // prevents false-sharing between full and empty
 	empty lfstack          // lock-free list of empty blocks workbuf
-	_     cpu.CacheLinePad // prevents false-sharing between empty and nproc/nwait
+	_     cpu.CacheLinePad // prevents false-sharing between empty and wbufSpans
 
 	wbufSpans struct {
 		lock mutex
@@ -337,12 +337,24 @@ type workType struct {
 		// one of the workbuf lists.
 		busy mSpanList
 	}
-	_ cpu.CacheLinePad // prevents false-sharing between wbufSpans and spanq
+	_ cpu.CacheLinePad // prevents false-sharing between wbufSpans and spanWorkMask
 
-	// Global queue of spans to scan.
+	// spanqMask is a bitmap indicating which Ps have local work worth stealing.
+	// Set or cleared by the owning P, cleared by stealing Ps.
+	//
+	// spanqMask is like a proxy for a global queue. An important invariant is that
+	// forced flushing like gcw.dispose must set this bit on any P that has local
+	// span work.
+	spanqMask pMask
+	_         cpu.CacheLinePad // prevents false-sharing between spanqMask and everything else
+
+	// List of all spanSPMCs.
 	//
 	// Only used if goexperiment.GreenTeaGC.
-	spanq spanQueue
+	spanSPMCs struct {
+		lock mutex // no lock rank because it's a leaf lock (see mklockrank.go).
+		all  *spanSPMC
+	}
 
 	// Restore 64-bit alignment on 32-bit.
 	// _ uint32
@@ -711,8 +723,9 @@ func gcStart(trigger gcTrigger) {
 		traceRelease(trace)
 	}
 
-	// Check that all Ps have finished deferred mcache flushes.
+	// Check and setup per-P state.
 	for _, p := range allp {
+		// Check that all Ps have finished deferred mcache flushes.
 		if fg := p.mcache.flushGen.Load(); fg != mheap_.sweepgen {
 			println("runtime: p", p.id, "flushGen", fg, "!= sweepgen", mheap_.sweepgen)
 			throw("p mcache not flushed")
@@ -923,6 +936,7 @@ top:
 		// TODO(austin): Break up these workbufs to
 		// better distribute work.
 		pp.gcw.dispose()
+
 		// Collect the flushedWork flag.
 		if pp.gcw.flushedWork {
 			atomic.Xadd(&gcMarkDoneFlushed, 1)
@@ -1623,17 +1637,6 @@ func gcEndWork() (last bool) {
 	return incnwait == work.nproc && !gcMarkWorkAvailable()
 }
 
-// gcMarkWorkAvailable reports whether there's any non-local work available to do.
-func gcMarkWorkAvailable() bool {
-	if !work.full.empty() || !work.spanq.empty() {
-		return true // global work available
-	}
-	if work.markrootNext < work.markrootJobs {
-		return true // root scan work available
-	}
-	return false
-}
-
 // gcMark runs the mark (or, for concurrent GC, mark termination)
 // All gcWork caches must be empty.
 // STW is in effect at this point.
@@ -1644,8 +1647,8 @@ func gcMark(startTime int64) {
 	work.tstart = startTime
 
 	// Check that there's no marking work remaining.
-	if work.full != 0 || work.markrootNext < work.markrootJobs || !work.spanq.empty() {
-		print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, " spanq.n=", work.spanq.size(), "\n")
+	if work.full != 0 || work.markrootNext < work.markrootJobs {
+		print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n")
 		panic("non-empty mark queue after concurrent mark")
 	}
 
@@ -1761,10 +1764,12 @@ func gcSweep(mode gcMode) bool {
 		// Sweep all spans eagerly.
 		for sweepone() != ^uintptr(0) {
 		}
-		// Free workbufs eagerly.
+		// Free workbufs and span rings eagerly.
 		prepareFreeWorkbufs()
 		for freeSomeWbufs(false) {
 		}
+		for freeSomeSpanSPMCs(false) {
+		}
 		// All "free" events for this mark/sweep cycle have
 		// now happened, so we can make this profile cycle
 		// available immediately.