Merge pull request #929 from njhale/pprof-option

Add optional profiling

Author: openshift-merge-robot

Documentation/design/profiling.md

@@ -0,0 +1,43 @@
+# Profiling OLM Operators
+
+OLM's `olm` and `catalog` commands support serving profiling samples via the `--profiling` option.
+
+```sh
+# run olm operator with profiling enabled
+$ go run cmd/olm/main.go --profiling --kubeconfig ~/.kube/config
+```
+
+Samples are in a format recognized by [pprof](https://golang.org/pkg/net/http/pprof) and an index of available profile types is made available at `https://127.0.0.1:8080/debug/pprof`.
+
+If profiling is enabled, but operators are running on a kubernetes cluster, a convienient way to expose samples locally is with `kubectl port-forward`:
+
+```sh
+# forward traffic from 127.0.0.1:8080 to port 8080 of catalog operator pods
+$ kubectl -n <olm-namespace> port-forward deployments/catalog-operator
+```
+
+When profiling is enabled, `go tool pprof` can be used to export and visualize samples:
+
+```sh
+# assuming a catalog operator's samples are accessible at 127.0.0.1:8080:
+# show in-use heap memory in top format
+$ go tool pprof -top http://127.0.0.1:8080/debug/pprof/heap
+Fetching profile over HTTP from http://127.0.0.1:8080/debug/pprof/heap
+Saved profile in /Users/nhale/pprof/pprof.catalog.alloc_objects.alloc_space.inuse_objects.inuse_space.013.pb.gz
+File: catalog
+Type: inuse_space
+Time: Jun 27, 2019 at 12:27pm (EDT)
+Showing nodes accounting for 2202.74kB, 100% of 2202.74kB total
+      flat  flat%   sum%        cum   cum%
+  650.62kB 29.54% 29.54%   650.62kB 29.54%  bufio.NewWriterSize
+  520.04kB 23.61% 53.15%   520.04kB 23.61%  golang.org/x/net/http2.NewFramer.func1
+  520.04kB 23.61% 76.75%   520.04kB 23.61%  sync.(*Map).LoadOrStore
+  512.03kB 23.25%   100%   512.03kB 23.25%  github.com/modern-go/reflect2.newUnsafeStructField
+  ...
+
+# save in-use objects graph to svg file
+$ go tool pprof -sample_index=inuse_objects -svg http://127.0.0.1:8080/debug/pprof/heap
+Fetching profile over HTTP from http://127.0.0.1:8080/debug/pprof/heap
+Saved profile in /Users/<user>/pprof/pprof.catalog.alloc_objects.alloc_space.inuse_objects.inuse_space.01.pb.gz
+Generating report in profile001.svg
+```

cmd/catalog/main.go

@@ -10,15 +10,16 @@ import (
 	"time"
 
 	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	utilclock "k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/client-go/tools/clientcmd"
 
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/catalog"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
@@ -61,6 +62,9 @@ var (
 
 	tlsCertPath = flag.String(
 		"tls-cert", "", "Path to use for certificate key (requires tls-key)")
+
+	profiling = flag.Bool(
+		"profiling", false, "serve profiling data (on port 8080)")
 )
 
 func init() {
@@ -114,6 +118,13 @@ func main() {
 	healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	})
+
+	// Serve profiling if enabled
+	if *profiling {
+		logger.Infof("profiling enabled")
+		profile.RegisterHandlers(healthMux)
+	}
+
 	go http.ListenAndServe(":8080", healthMux)
 
 	metricsMux := http.NewServeMux()

cmd/olm/main.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	configv1client "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
@@ -19,6 +18,8 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/api/client"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorclient"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorstatus"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/signals"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/metrics"
 	olmversion "github.com/operator-framework/operator-lifecycle-manager/pkg/version"
@@ -59,6 +60,9 @@ var (
 
 	tlsCertPath = flag.String(
 		"tls-cert", "", "Path to use for certificate key (requires tls-key)")
+
+	profiling = flag.Bool(
+		"profiling", false, "serve profiling data (on port 8080)")
 )
 
 func init() {
@@ -114,6 +118,13 @@ func main() {
 	healthMux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 	})
+
+	// Serve profiling if enabled
+	if *profiling {
+		logger.Infof("profiling enabled")
+		profile.RegisterHandlers(healthMux)
+	}
+
 	go func() {
 		err := http.ListenAndServe(":8080", healthMux)
 		if err != nil {

pkg/lib/profile/profile.go

@@ -0,0 +1,64 @@
+package profile
+
+import (
+	"net/http"
+	"net/http/pprof"
+)
+
+type profileConfig struct {
+	pprof   bool
+	cmdline bool
+	profile bool
+	symbol  bool
+	trace   bool
+}
+
+// Option applies a configuration option to the given config.
+type Option func(p *profileConfig)
+
+func (p *profileConfig) apply(options []Option) {
+	if len(options) == 0 {
+		// If no options are given, default to all
+		p.pprof = true
+		p.cmdline = true
+		p.profile = true
+		p.symbol = true
+		p.trace = true
+
+		return
+	}
+
+	for _, o := range options {
+		o(p)
+	}
+}
+
+func defaultProfileConfig() *profileConfig {
+	// Initialize config
+	return &profileConfig{}
+}
+
+// RegisterHandlers registers profile Handlers with the given ServeMux.
+//
+// The Handlers registered are determined by the given options.
+// If no options are given, all available handlers are registered by default.
+func RegisterHandlers(mux *http.ServeMux, options ...Option) {
+	config := defaultProfileConfig()
+	config.apply(options)
+
+	if config.pprof {
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+	}
+	if config.cmdline {
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	}
+	if config.profile {
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	}
+	if config.symbol {
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	}
+	if config.trace {
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+	}
+}