Switched markrootNext and markrootJobs to atomic.Uint32. Added GCDEBUG flag that makes the GC continuously run leak detection.

Author: VladSaiocUber

src/runtime/mgc.go

@@ -364,8 +364,8 @@ type workType struct {
 	// (and thus 8-byte alignment even on 32-bit architectures).
 	bytesMarked uint64
 
-	markrootNext uint32 // next markroot job
-	markrootJobs uint32 // number of markroot jobs
+	markrootNext atomic.Uint32 // next markroot job
+	markrootJobs atomic.Uint32 // number of markroot jobs
 
 	nproc  uint32
 	tstart int64
@@ -734,7 +734,8 @@ func gcStart(trigger gcTrigger) {
 	} else if debug.gcstoptheworld == 2 {
 		mode = gcForceBlockMode
 	} else if goexperiment.GoroutineLeakFinderGC {
-		if work.goroutineLeakFinder.pending.Load() {
+		if work.goroutineLeakFinder.pending.Load() ||
+			debug.gcgoroutineleaks > 0 {
 			// Fully stop the world if running deadlock detection.
 			mode = gcForceBlockMode
 		}
@@ -816,7 +817,8 @@ func gcStart(trigger gcTrigger) {
 		schedEnableUser(false)
 	}
 
-	if work.goroutineLeakFinder.pending.Load() {
+	if work.goroutineLeakFinder.pending.Load() ||
+		debug.gcgoroutineleaks > 0 {
 		work.goroutineLeakFinder.enabled = true
 		work.goroutineLeakFinder.pending.Store(false)
 		gcUntrackSyncObjects()
@@ -1170,13 +1172,11 @@ func gcDiscoverMoreStackRoots() {
 		}
 	}
 
-	var oldRootJobs int32 = int32(atomic.Load(&work.markrootJobs))
 	var newRootJobs int32 = int32(work.baseStacks) + int32(vIndex)
-
-	if newRootJobs > oldRootJobs {
+	if newRootJobs > int32(work.markrootJobs.Load()) {
 		// reset markrootNext as it could have been incremented past markrootJobs
 		work.nLiveStackRoots = vIndex
-		atomic.Store(&work.markrootJobs, uint32(newRootJobs))
+		work.markrootJobs.Store(uint32(newRootJobs))
 	}
 }
 
@@ -1237,7 +1237,7 @@ func findGoleaks() bool {
 		work.stackRoots[work.nLiveStackRoots] = gp
 		work.nLiveStackRoots += 1
 		// We now have one more markroot job.
-		work.markrootJobs += 1
+		work.markrootJobs.Add(1)
 		// We might still have some work to do.
 		// Make sure in the next iteration we will check re-check for new runnable goroutines.
 		foundMoreWork = true
@@ -1261,7 +1261,7 @@ func findGoleaks() bool {
 		println()
 	}
 	// Put the remaining roots as ready for marking and drain them.
-	work.markrootJobs += uint32(work.nStackRoots - work.nLiveStackRoots)
+	work.markrootJobs.Add(int32(work.nStackRoots - work.nLiveStackRoots))
 	work.nLiveStackRoots = work.nStackRoots
 	return true
 }
@@ -1854,9 +1854,7 @@ func gcMarkWorkAvailable(p *p) bool {
 	if !work.full.empty() || !work.spanq.empty() {
 		return true // global work available
 	}
-	rootNext := atomic.Load(&work.markrootNext)
-	rootJobs := atomic.Load(&work.markrootJobs)
-	if rootNext < rootJobs {
+	if work.markrootNext.Load() < work.markrootJobs.Load() {
 		return true // root scan work available
 	}
 	return false
@@ -1872,10 +1870,8 @@ func gcMark(startTime int64) {
 	work.tstart = startTime
 
 	// Check that there's no marking work remaining.
-	rootNext := atomic.Load(&work.markrootNext)
-	rootJobs := atomic.Load(&work.markrootJobs)
-	if work.full != 0 || rootNext < rootJobs {
-		print("runtime: full=", hex(work.full), " next=", rootNext, " jobs=", rootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n")
+	if work.full != 0 || work.markrootNext.Load() < work.markrootJobs.Load() {
+		print("runtime: full=", hex(work.full), " next=", work.markrootNext.Load(), " jobs=", work.markrootJobs.Load(), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n")
 		panic("non-empty mark queue after concurrent mark")
 	}
 

src/runtime/mgcmark.go

@@ -162,8 +162,8 @@ func gcPrepareMarkRoots() {
 
 	work.nStackRoots = len(work.stackRoots)
 
-	work.markrootNext = 0
-	work.markrootJobs = uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nLiveStackRoots)
+	work.markrootNext.Store(0)
+	work.markrootJobs.Store(uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nLiveStackRoots))
 
 	// Calculate base indexes of each root type
 	work.baseData = uint32(fixedRootCount)
@@ -176,10 +176,8 @@ func gcPrepareMarkRoots() {
 // gcMarkRootCheck checks that all roots have been scanned. It is
 // purely for debugging.
 func gcMarkRootCheck() {
-	rootNext := atomic.Load(&work.markrootNext)
-	rootJobs := atomic.Load(&work.markrootJobs)
-	if rootNext < rootJobs {
-		print(rootNext, " of ", rootJobs, " markroot jobs done\n")
+	if work.markrootNext.Load() < work.markrootJobs.Load() {
+		print(work.markrootNext.Load(), " of ", work.markrootJobs.Load(), " markroot jobs done\n")
 		throw("left over markroot jobs")
 	}
 
@@ -1197,21 +1195,19 @@ func gcDrainMarkWorkerFractional(gcw *gcWork) {
 
 func gcUpdateMarkrootNext() (uint32, bool) {
 	var success bool
-	var next uint32 = atomic.Load(&work.markrootNext)
-	var jobs uint32 = atomic.Load(&work.markrootJobs)
+	next, jobs := work.markrootNext.Load(), work.markrootJobs.Load()
 
 	if next < jobs {
 		// still work available at the moment
 		for !success {
-			success = atomic.Cas(&work.markrootNext, next, next+1)
+			success = work.markrootNext.CompareAndSwap(next, next+1)
 			// We manage to snatch a root job. Return the root index.
 			if success {
 				return next, true
 			}
 
 			// Get the latest value of markrootNext.
-			next = atomic.Load(&work.markrootNext)
-			jobs := atomic.Load(&work.markrootJobs)
+			next = work.markrootNext.Load()
 			// We are out of markroot jobs.
 			if next >= jobs {
 				break
@@ -1279,9 +1275,7 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) {
 		}
 	}
 
-	rootNext := atomic.Load(&work.markrootNext)
-	rootJobs := atomic.Load(&work.markrootJobs)
-	if rootNext < rootJobs {
+	if work.markrootNext.Load() < work.markrootJobs.Load() {
 		// Stop if we're preemptible, if someone wants to STW, or if
 		// someone is calling forEachP.
 		for !(gp.preempt && (preemptible || sched.gcwaiting.Load() || pp.runSafePointFn != 0)) {
@@ -1432,7 +1426,7 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 {
 					wbBufFlush()
 					if b = gcw.tryGetObj(); b == 0 {
 						// Try to do a root job.
-						if atomic.Load(&work.markrootNext) < atomic.Load(&work.markrootJobs) {
+						if work.markrootNext.Load() < work.markrootJobs.Load() {
 							job, success := gcUpdateMarkrootNext()
 							if success {
 								workFlushed += markroot(gcw, job, false)

src/runtime/runtime1.go

@@ -316,6 +316,7 @@ var debug struct {
 	dontfreezetheworld       int32
 	efence                   int32
 	gccheckmark              int32
+	gcgoroutineleaks         int32
 	gcpacertrace             int32
 	gcshrinkstackoff         int32
 	gcstoptheworld           int32
@@ -381,6 +382,7 @@ var dbgvars = []*dbgVar{
 	{name: "efence", value: &debug.efence},
 	{name: "gccheckmark", value: &debug.gccheckmark},
 	{name: "gcpacertrace", value: &debug.gcpacertrace},
+	{name: "gcgoroutineleaks", value: &debug.gcgoroutineleaks},
 	{name: "gcshrinkstackoff", value: &debug.gcshrinkstackoff},
 	{name: "gcstoptheworld", value: &debug.gcstoptheworld},
 	{name: "gctrace", value: &debug.gctrace},