Test for goroutine leak GC.

Author: VladSaiocUber

src/runtime/gc_test.go

@@ -0,0 +1,133 @@
+/*
+ * Project: cockroach
+ * Issue or PR  : https://github.com/cockroachdb/cockroach/pull/10214
+ * Buggy version: 7207111aa3a43df0552509365fdec741a53f873f
+ * fix commit-id: 27e863d90ab0660494778f1c35966cc5ddc38e32
+ * Flaky: 3/100
+ * Description: This deadlock is caused by different order when acquiring
+ * coalescedMu.Lock() and raftMu.Lock(). The fix is to refactor sendQueuedHeartbeats()
+ * so that cockroachdb can unlock coalescedMu before locking raftMu.
+ */
+package main
+
+import (
+	"runtime"
+	"sync"
+	"time"
+	"unsafe"
+)
+
+func init() {
+	register("Cockroach10214", Cockroach10214)
+}
+
+type Store_cockroach10214 struct {
+	coalescedMu struct {
+		sync.Mutex
+		heartbeatResponses []int
+	}
+	mu struct {
+		replicas map[int]*Replica_cockroach10214
+	}
+}
+
+func (s *Store_cockroach10214) sendQueuedHeartbeats() {
+	s.coalescedMu.Lock() // LockA acquire
+	runtime.Gosched()
+	defer s.coalescedMu.Unlock()
+	for i := 0; i < len(s.coalescedMu.heartbeatResponses); i++ {
+		s.sendQueuedHeartbeatsToNode() // LockB
+	}
+	// LockA release
+}
+
+func (s *Store_cockroach10214) sendQueuedHeartbeatsToNode() {
+	for i := 0; i < len(s.mu.replicas); i++ {
+		r := s.mu.replicas[i]
+		r.reportUnreachable() // LockB
+	}
+}
+
+type Replica_cockroach10214 struct {
+	raftMu sync.Mutex
+	mu     sync.Mutex
+	store  *Store_cockroach10214
+}
+
+func (r *Replica_cockroach10214) reportUnreachable() {
+	r.raftMu.Lock() // LockB acquire
+	runtime.Gosched()
+	//+time.Sleep(time.Nanosecond)
+	defer r.raftMu.Unlock()
+	// LockB release
+}
+
+func (r *Replica_cockroach10214) tick() {
+	r.raftMu.Lock() // LockB acquire
+	runtime.Gosched()
+	defer r.raftMu.Unlock()
+	r.tickRaftMuLocked()
+	// LockB release
+}
+
+func (r *Replica_cockroach10214) tickRaftMuLocked() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.maybeQuiesceLocked() {
+		return
+	}
+}
+func (r *Replica_cockroach10214) maybeQuiesceLocked() bool {
+	for i := 0; i < 2; i++ {
+		if !r.maybeCoalesceHeartbeat() {
+			return true
+		}
+	}
+	return false
+}
+func (r *Replica_cockroach10214) maybeCoalesceHeartbeat() bool {
+	msgtype := uintptr(unsafe.Pointer(r)) % 3
+	switch msgtype {
+	case 0, 1, 2:
+		r.store.coalescedMu.Lock() // LockA acquire
+	default:
+		return false
+	}
+	r.store.coalescedMu.Unlock() // LockA release
+	return true
+}
+
+func Cockroach10214() {
+	defer func() {
+		time.Sleep(100 * time.Millisecond)
+		runtime.GC()
+	}()
+	for i := 0; i < 1000; i++ {
+		go func() {
+			store := &Store_cockroach10214{}
+			responses := &store.coalescedMu.heartbeatResponses
+			*responses = append(*responses, 1, 2)
+			store.mu.replicas = make(map[int]*Replica_cockroach10214)
+
+			rp1 := &Replica_cockroach10214{
+				store: store,
+			}
+			rp2 := &Replica_cockroach10214{
+				store: store,
+			}
+			store.mu.replicas[0] = rp1
+			store.mu.replicas[1] = rp2
+
+			go func() {
+				// deadlocks: x > 0
+				store.sendQueuedHeartbeats()
+			}()
+
+			go func() {
+				// deadlocks: x > 0
+				rp1.tick()
+			}()
+
+		}()
+	}
+}

src/runtime/testdata/testgoroutineleakgc/cockroach10214.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+func init() {
+	register("Cockroach1055", Cockroach1055)
+}
+
+type Stopper_cockroach1055 struct {
+	stopper  chan struct{}
+	stop     sync.WaitGroup
+	mu       sync.Mutex
+	draining int32
+	drain    sync.WaitGroup
+}
+
+func (s *Stopper_cockroach1055) AddWorker() {
+	s.stop.Add(1)
+}
+
+func (s *Stopper_cockroach1055) ShouldStop() <-chan struct{} {
+	if s == nil {
+		return nil
+	}
+	return s.stopper
+}
+
+func (s *Stopper_cockroach1055) SetStopped() {
+	if s != nil {
+		s.stop.Done()
+	}
+}
+
+func (s *Stopper_cockroach1055) Quiesce() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.draining = 1
+	s.drain.Wait()
+	s.draining = 0
+}
+
+func (s *Stopper_cockroach1055) Stop() {
+	s.mu.Lock() // L1
+	defer s.mu.Unlock()
+	atomic.StoreInt32(&s.draining, 1)
+	s.drain.Wait()
+	close(s.stopper)
+	s.stop.Wait()
+}
+
+func (s *Stopper_cockroach1055) StartTask() bool {
+	if atomic.LoadInt32(&s.draining) == 0 {
+		s.mu.Lock()
+		defer s.mu.Unlock()
+		s.drain.Add(1)
+		return true
+	}
+	return false
+}
+
+func NewStopper_cockroach1055() *Stopper_cockroach1055 {
+	return &Stopper_cockroach1055{
+		stopper: make(chan struct{}),
+	}
+}
+
+func Cockroach1055() {
+	defer func() {
+		time.Sleep(1 * time.Second)
+		runtime.GC()
+	}()
+
+	for i := 0; i <= 1000; i++ {
+		go func() { // G1
+			// deadlocks: x > 0
+			var stoppers []*Stopper_cockroach1055
+			for i := 0; i < 2; i++ {
+				stoppers = append(stoppers, NewStopper_cockroach1055())
+			}
+
+			for i := range stoppers {
+				s := stoppers[i]
+				s.AddWorker()
+				go func() { // G2
+					// deadlocks: x > 0
+					s.StartTask()
+					<-s.ShouldStop()
+					s.SetStopped()
+				}()
+			}
+
+			done := make(chan struct{})
+			go func() { // G3
+				// deadlocks: x > 0
+				for _, s := range stoppers {
+					s.Quiesce()
+				}
+				for _, s := range stoppers {
+					s.Stop()
+				}
+				close(done)
+			}()
+
+			<-done
+		}()
+	}
+}
+
+//	Example deadlock trace:
+//
+//	G1						G2.0						G2.1						G2.2						G3
+//	---------------------------------------------------------------------------------------------------------------------
+//	s[0].stop.Add(1) [1]
+//	go func() [G2.0]
+//	s[1].stop.Add(1) [1]	.
+//	go func() [G2.1]		.
+//	s[2].stop.Add(1) [1]	.							.
+//	go func() [G2.2]		.							.
+//	go func() [G3]			.							.							.
+//	<-done					.							.							.							.
+//	.						s[0].StartTask()			.							.							.
+//	.						s[0].draining == 0			.							.							.
+//	.						.							s[1].StartTask()			.							.
+//	.						.							s[1].draining == 0			.							.
+//	.						.							.							s[2].StartTask()			.
+//	.						.							.							s[2].draining == 0			.
+//	.						.							.							.							s[0].Quiesce()
+//	.						.							.							.							s[0].mu.Lock() [L1[0]]
+//	.						s[0].mu.Lock() [L1[0]]		.							.							.
+//	.						s[0].drain.Add(1) [1]		.							.							.
+//	.						s[0].mu.Unlock() [L1[0]]	.							.							.
+//	.						<-s[0].ShouldStop()			.							.							.
+//	.						.							.							.							s[0].draining = 1
+//	.						.							.							.							s[0].drain.Wait()
+//	.						.							s[0].mu.Lock() [L1[1]]		.							.
+//	.						.							s[1].drain.Add(1) [1]		.							.
+//	.						.							s[1].mu.Unlock() [L1[1]]	.							.
+//	.						.							<-s[1].ShouldStop()			.							.
+//	.						.							.							s[2].mu.Lock() [L1[2]]		.
+//	.						.							.							s[2].drain.Add() [1]		.
+//	.						.							.							s[2].mu.Unlock() [L1[2]]	.
+//	.						.							.							<-s[2].ShouldStop()			.
+//	----------------------------------------------------G1, G2.[0..2], G3 leak------------------------------------------------