Count failed API calls.

This adds an additional metric for counting failed API calls to the Git
hosting service.

A failed call is one that causes either an error, or with the response
status >= 400.

Author: bigkevmcd

pkg/git/client.go

@@ -55,12 +55,16 @@ func (c *SCMClient) FileContents(ctx context.Context, repo, path, ref string) ([
 func (c *SCMClient) CreateStatus(ctx context.Context, repo, commit string, s *scm.StatusInput) error {
 	c.m.CountAPICall("create_status")
 	_, r, err := c.client.Repositories.CreateStatus(ctx, repo, commit, s)
-	if isErrorStatus(r.Status) {
+	errResponse := isErrorStatus(r.Status)
+	if errResponse || err != nil {
+		c.m.CountFailedAPICall("file_contents")
+	}
+	if errResponse {
 		return scmError{msg: fmt.Sprintf("failed to create commitstatus in repo %s commit %s", repo, commit), Status: r.Status}
 	}
 	return err
 }
 
 func isErrorStatus(i int) bool {
-	return i > 400
+	return i >= 400
 }

pkg/git/client_test.go

@@ -117,13 +117,14 @@ func TestCreateStatus(t *testing.T) {
 }
 
 func TestCreateStatusWithNotFoundResponse(t *testing.T) {
+	m := metrics.NewMock()
 	as := makeAPIServer(t, "/api/v3/repos/Codertocat/Hello-World/statuses/6dcb09b5b57875f334f61aebed695e2e4193db5e", "", "")
 	defer as.Close()
 	scmClient, err := factory.NewClient("github", as.URL, "", factory.Client(as.Client()))
 	if err != nil {
 		t.Fatal(err)
 	}
-	client := New(scmClient, nil, metrics.NewMock())
+	client := New(scmClient, nil, m)
 
 	status := &scm.StatusInput{
 		State: scm.StatePending,
@@ -135,6 +136,9 @@ func TestCreateStatusWithNotFoundResponse(t *testing.T) {
 	if !IsNotFound(err) {
 		t.Fatal(err)
 	}
+	if m.FailedAPICalls != 1 {
+		t.Fatalf("metrics count of failed API calls, got %d, want 1", m.FailedAPICalls)
+	}
 }
 
 func makeAPIServer(t *testing.T, urlPath, ref, fixture string) *httptest.Server {

pkg/metrics/interface.go

@@ -12,4 +12,7 @@ type Interface interface {
 
 	// CountAPICall records API calls to the upstream hosting service.
 	CountAPICall(name string)
+
+	// CountFailedAPICall records failed API calls to the upstream hosting service.
+	CountFailedAPICall(name string)
 }

pkg/metrics/metrics.go

@@ -8,9 +8,10 @@ import (
 // PrometheusMetrics is a wrapper around Prometheus metrics for counting
 // events in the system.
 type PrometheusMetrics struct {
-	hooks        *prometheus.CounterVec
-	invalidHooks prometheus.Counter
-	apiCalls     *prometheus.CounterVec
+	hooks          *prometheus.CounterVec
+	invalidHooks   prometheus.Counter
+	apiCalls       *prometheus.CounterVec
+	failedAPICalls *prometheus.CounterVec
 }
 
 // New creates and returns a PrometheusMetrics initialised with prometheus
@@ -39,9 +40,16 @@ func New(ns string, reg prometheus.Registerer) *PrometheusMetrics {
 		Help:      "Count of API Calls made",
 	}, []string{"kind"})
 
+	pm.failedAPICalls = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: ns,
+		Name:      "failed_api_calls_total",
+		Help:      "Count of failed API Calls made",
+	}, []string{"kind"})
+
 	reg.MustRegister(pm.hooks)
 	reg.MustRegister(pm.invalidHooks)
 	reg.MustRegister(pm.apiCalls)
+	reg.MustRegister(pm.failedAPICalls)
 	return pm
 }
 
@@ -59,3 +67,8 @@ func (m *PrometheusMetrics) CountInvalidHook() {
 func (m *PrometheusMetrics) CountAPICall(name string) {
 	m.apiCalls.With(prometheus.Labels{"kind": name}).Inc()
 }
+
+// CountFailedAPICall records failled outgoing API calls to upstream services.
+func (m *PrometheusMetrics) CountFailedAPICall(name string) {
+	m.failedAPICalls.With(prometheus.Labels{"kind": name}).Inc()
+}

pkg/metrics/metrics_test.go

@@ -55,3 +55,17 @@ dsl_api_calls_total{kind="file_contents"} 1
 		t.Fatal(err)
 	}
 }
+
+func TestCountFailedAPICall(t *testing.T) {
+	m := New("dsl", prometheus.NewRegistry())
+	m.CountFailedAPICall("commit_status")
+
+	err := testutil.CollectAndCompare(m.failedAPICalls, strings.NewReader(`
+# HELP dsl_failed_api_calls_total Count of failed API Calls made
+# TYPE dsl_failed_api_calls_total counter
+dsl_failed_api_calls_total{kind="commit_status"} 1
+`))
+	if err != nil {
+		t.Fatal(err)
+	}
+}

pkg/metrics/mock.go

@@ -4,12 +4,15 @@ import (
 	"github.com/jenkins-x/go-scm/scm"
 )
 
-// MockMetrics is a value that provides a wrapper around Mock
-// metrics for counting events in the system.
+var _ Interface = (*MockMetrics)(nil)
+
+// MockMetrics is a type that provides a simple counter for metrics for test
+// purposes.
 type MockMetrics struct {
-	Hooks        int
-	InvalidHooks int
-	APICalls     int
+	Hooks          int
+	InvalidHooks   int
+	APICalls       int
+	FailedAPICalls int
 }
 
 // NewMock creates and returns a MockMetrics.
@@ -31,3 +34,8 @@ func (m *MockMetrics) CountInvalidHook() {
 func (m *MockMetrics) CountAPICall(name string) {
 	m.APICalls++
 }
+
+// CountFailedAPICall records failed outgoing API calls to upstream services.
+func (m *MockMetrics) CountFailedAPICall(name string) {
+	m.FailedAPICalls++
+}