Add BenchmarkProfilerGoroutines and update README

Author: felixge

BenchmarkProfilerGoroutines.txt

@@ -0,0 +1,25 @@
+$ go test -bench=BenchmarkProfilerGoroutines
+goos: darwin
+goarch: amd64
+pkg: github.com/felixge/fgprof
+BenchmarkProfilerGoroutines/1_goroutines-8         	  43431	    26860 ns/op
+BenchmarkProfilerGoroutines/2_goroutines-8         	  42590	    27648 ns/op
+BenchmarkProfilerGoroutines/4_goroutines-8         	  40725	    28694 ns/op
+BenchmarkProfilerGoroutines/8_goroutines-8         	  37874	    31067 ns/op
+BenchmarkProfilerGoroutines/16_goroutines-8        	  32778	    37302 ns/op
+BenchmarkProfilerGoroutines/32_goroutines-8        	  25447	    47171 ns/op
+BenchmarkProfilerGoroutines/64_goroutines-8        	  17937	    66803 ns/op
+BenchmarkProfilerGoroutines/128_goroutines-8       	  11138	   108283 ns/op
+BenchmarkProfilerGoroutines/256_goroutines-8       	   5232	   191830 ns/op
+BenchmarkProfilerGoroutines/512_goroutines-8       	   2848	   351686 ns/op
+BenchmarkProfilerGoroutines/1024_goroutines-8      	   1611	   681412 ns/op
+BenchmarkProfilerGoroutines/2048_goroutines-8      	    846	  1396125 ns/op
+BenchmarkProfilerGoroutines/4096_goroutines-8      	    358	  3286943 ns/op
+BenchmarkProfilerGoroutines/8192_goroutines-8      	    153	  7813804 ns/op
+BenchmarkProfilerGoroutines/16384_goroutines-8     	     70	 16440643 ns/op
+BenchmarkProfilerGoroutines/32768_goroutines-8     	     33	 34101649 ns/op
+BenchmarkProfilerGoroutines/65536_goroutines-8     	     16	 68460458 ns/op
+BenchmarkProfilerGoroutines/131072_goroutines-8    	      8	134481118 ns/op
+BenchmarkProfilerGoroutines/262144_goroutines-8    	      4	270522885 ns/op
+BenchmarkProfilerGoroutines/524288_goroutines-8    	      2	567821104 ns/op
+BenchmarkProfilerGoroutines/1048576_goroutines-8   	      1	1202184643 ns/op

README.md

@@ -175,7 +175,7 @@ fgprof is implemented as a background goroutine that wakes up 99 times per secon
 
 This data is used to maintain an in-memory stack counter which can be converted to the pprof or folded output format. The meat of the implementation is super simple and < 100 lines of code, you should [check it out](./fgprof.go).
 
-Generally speaking, fgprof should not have a big impact on the performance of your program. However `runtime.GoroutineProfile` calls `stopTheWorld()` and could be slow if you have a lot of goroutines. For now the advise is to test the impact of the profiler on a development environment before running it against production instances. In the future this README will try to provide a more detailed analysis of the performance impact.
+The overhead of fgprof increases with the number of active goroutines (including those waiting on I/O, Channels, Locks, etc.) executed by your program. If your program typically has less than 1000 active goroutines, you shouldn't have much to worry about. However, at 10k or more goroutines fgprof is likely to become unable to maintain its sampling rate and to significantly degrade the performance of your application, see [BenchmarkProfilerGoroutines.txt](./BenchmarkProfilerGoroutines.txt). The latter is due to `runtime.GoroutineProfile()` calling `stopTheWorld()`. For now the advise is to test the impact of the profiler on a development environment before running it against production instances. There are ideas for making fgprof more scalable and safe for programs with a high number of goroutines, but they will likely need improved APIs from the Go core.
 
 ### Go's builtin CPU Profiler
 

fgprof_test.go

@@ -2,6 +2,7 @@ package fgprof
 
 import (
 	"bytes"
+	"fmt"
 	"strings"
 	"testing"
 	"time"
@@ -31,6 +32,51 @@ func BenchmarkProfiler(b *testing.B) {
 	}
 }
 
+func BenchmarkProfilerGoroutines(b *testing.B) {
+	for g := 1; g <= 1024*1024; g = g * 2 {
+		g := g
+		name := fmt.Sprintf("%d goroutines", g)
+
+		b.Run(name, func(b *testing.B) {
+			prof := &profiler{}
+			initalRoutines := len(prof.GoroutineProfile())
+
+			readyCh := make(chan struct{})
+			stopCh := make(chan struct{})
+			for i := 0; i < g; i++ {
+				go func() {
+					defer func() { stopCh <- struct{}{} }()
+					readyCh <- struct{}{}
+				}()
+				<-readyCh
+			}
+
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				stacks := prof.GoroutineProfile()
+				gotRoutines := len(stacks) - initalRoutines
+				if gotRoutines != g {
+					b.Logf("want %d goroutines, but got %d on iteration %d", g, len(stacks), i)
+				}
+			}
+			b.StopTimer()
+			for i := 0; i < g; i++ {
+				<-stopCh
+			}
+			start := time.Now()
+			for i := 0; ; i++ {
+				if len(prof.GoroutineProfile()) == initalRoutines {
+					break
+				}
+				time.Sleep(20 * time.Millisecond)
+				if time.Since(start) > 10*time.Second {
+					b.Fatalf("%d goroutines still running, want %d", len(prof.GoroutineProfile()), initalRoutines)
+				}
+			}
+		})
+	}
+}
+
 func BenchmarkStackCounter(b *testing.B) {
 	prof := &profiler{}
 	stacks := prof.GoroutineProfile()