Add event.dropped and event.age metrics (#52)

These provide more insight into the queued asynchronous scheduler.

Author: bluekeyes

README.md

@@ -13,7 +13,7 @@ logic of your application.
 * [Asynchronous Dispatch](#asynchronous-dispatch)
 * [Structured Logging](#structured-logging)
 * [GitHub Clients](#github-clients)
-  + [Metrics](#metrics)
+* [Metrics](#metrics)
 * [Background Jobs and Multi-Organization Operations](#background-jobs-and-multi-organization-operations)
 * [OAuth2](#oauth2)
 * [Stability and Versioning Guarantees](#stability-and-versioning-guarantees)
@@ -203,7 +203,7 @@ baseHandler, err := githubapp.NewDefaultCachingClientCreator(
 )
 ```
 
-### Metrics
+## Metrics
 
 `go-githubapp` uses [rcrowley/go-metrics][] to provide metrics. GitHub clients
 emit the metrics below if configured with the `githubapp.ClientMetrics`
@@ -220,13 +220,15 @@ middleware.
 | `github.rate.limit[installation:<id>]` | `gauge` | the maximum number of requests permitted to make per hour, tagged with the installation id |
 | `github.rate.remaining[installation:<id>]` | `gauge` | the number of requests remaining in the current rate limit window, tagged with the installation id |
 
-If using [asynchronous dispatch](#asynchronous-dispatch) and the `githubapp.WithSchedulingMetrics` option
-is set, these metrics are emitted:
+When using [asynchronous dispatch](#asynchronous-dispatch), the
+`githubapp.WithSchedulingMetrics` option emits the following metrics:
 
 | metric name | type | definition |
 | ----------- | ---- | ---------- |
 | `github.event.queue` | `gauge` | the number of queued unprocessed event |
 | `github.event.workers` | `gauge` | the number of workers actively processing events |
+| `github.event.dropped` | `counter` | the number events dropped due to limited queue capacity |
+| `github.event.age` | `histogram` | the age (queue time) in milliseconds of events at processing time |
 
 Note that metrics need to be published in order to be useful. Several
 [publishing options][] are available or you can implement your own.

githubapp/scheduler.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"sync/atomic"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/rcrowley/go-metrics"
@@ -27,6 +28,14 @@ import (
 const (
 	MetricsKeyQueueLength   = "github.event.queued"
 	MetricsKeyActiveWorkers = "github.event.workers"
+	MetricsKeyEventAge      = "github.event.age"
+	MetricsKeyDroppedEvents = "github.event.dropped"
+)
+
+const (
+	// values from metrics.NewTimer, which match those used by UNIX load averages
+	histogramReservoirSize = 1028
+	histogramAlpha         = 0.015
 )
 
 var (
@@ -119,11 +128,16 @@ func WithSchedulingMetrics(r metrics.Registry) SchedulerOption {
 		metrics.NewRegisteredFunctionalGauge(MetricsKeyActiveWorkers, r, func() int64 {
 			return atomic.LoadInt64(&s.activeWorkers)
 		})
+
+		sample := metrics.NewExpDecaySample(histogramReservoirSize, histogramAlpha)
+		s.eventAge = metrics.NewRegisteredHistogram(MetricsKeyEventAge, r, sample)
+		s.dropped = metrics.NewRegisteredCounter(MetricsKeyDroppedEvents, r)
 	}
 }
 
 type queueDispatch struct {
 	ctx context.Context
+	t   time.Time
 	d   Dispatch
 }
 
@@ -134,6 +148,9 @@ type scheduler struct {
 
 	activeWorkers int64
 	queue         chan queueDispatch
+
+	eventAge metrics.Histogram
+	dropped  metrics.Counter
 }
 
 func (s *scheduler) safeExecute(ctx context.Context, d Dispatch) {
@@ -224,6 +241,9 @@ func QueueAsyncScheduler(queueSize int, workers int, opts ...SchedulerOption) Sc
 	for i := 0; i < workers; i++ {
 		go func() {
 			for d := range s.queue {
+				if s.eventAge != nil {
+					s.eventAge.Update(time.Since(d.t).Milliseconds())
+				}
 				s.safeExecute(d.ctx, d.d)
 			}
 		}()
@@ -238,8 +258,11 @@ type queueScheduler struct {
 
 func (s *queueScheduler) Schedule(ctx context.Context, d Dispatch) error {
 	select {
-	case s.queue <- queueDispatch{ctx: s.derive(ctx), d: d}:
+	case s.queue <- queueDispatch{ctx: s.derive(ctx), t: time.Now(), d: d}:
 	default:
+		if s.dropped != nil {
+			s.dropped.Inc(1)
+		}
 		return ErrCapacityExceeded
 	}
 	return nil