* Added sentry reports as an extra middlware (#316)

* Added some additional headers to the report filter
* Added panic handler, added sentry filter for the events

Author: Igor Komlew

.gitignore

@@ -27,3 +27,4 @@ files/
 /env
 /dist
 /tmp_charm
+.vscode/

charm/serial-vault/config.yaml

@@ -71,3 +71,7 @@ options:
         description: "Factory sync only"
         type: string
         default: ""
+    sentryDSN:
+        description: "Sentry data source name"
+        type: string
+        default: ""

charm/serial-vault/templates/settings.yaml

@@ -16,3 +16,4 @@ jwtSecret: "{{ jwtSecret }}"
 syncUrl: "{{ syncUrl }}"
 syncUser: "{{ syncUser }}"
 syncAPIKey: "{{ syncAPIKey }}"
+sentryDSN: "{{ sentryDSN }}"

cmd/serial-vault/main.go

@@ -27,10 +27,17 @@ import (
 	"github.com/CanonicalLtd/serial-vault/datastore"
 	"github.com/CanonicalLtd/serial-vault/service"
 	svlog "github.com/CanonicalLtd/serial-vault/service/log"
+	"github.com/CanonicalLtd/serial-vault/service/sentry"
 	logging "github.com/op/go-logging"
 )
 
-func init() {
+const appName = "serialvault"
+
+func initLogger() {
+	err := sentry.Init(datastore.Environ.Config.SentryDSN, appName, datastore.Environ.Config.Version)
+	if err != nil {
+		svlog.Errorf("sentry init error: %v", err)
+	}
 	svlog.InitLogger(logging.INFO)
 }
 
@@ -42,6 +49,7 @@ func main() {
 	if err != nil {
 		svlog.Fatalf("Error parsing the config file: %v", err)
 	}
+	initLogger()
 
 	// Open the connection to the local database
 	datastore.OpenSysDatabase(datastore.Environ.Config.Driver, datastore.Environ.Config.DataSource)

config/config.go

@@ -22,6 +22,7 @@ package config
 import (
 	"flag"
 	"io/ioutil"
+	"os"
 
 	"github.com/CanonicalLtd/serial-vault/service/log"
 
@@ -57,6 +58,7 @@ type Settings struct {
 	SyncURL        string `yaml:"syncUrl"`
 	SyncUser       string `yaml:"syncUser"`
 	SyncAPIKey     string `yaml:"syncAPIKey"`
+	SentryDSN      string `yaml:"sentryDSN"`
 }
 
 // SettingsFile is the path to the YAML configuration file
@@ -95,5 +97,9 @@ func ReadConfig(settings *Settings, filePath string) error {
 		ServiceMode = settings.Mode
 	}
 
+	if settings.SentryDSN == "" {
+		settings.SentryDSN = os.Getenv("SENTRY_DSN")
+	}
+
 	return nil
 }

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/Masterminds/squirrel v1.2.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/getsentry/sentry-go v0.7.0
 	github.com/godbus/dbus v4.1.0+incompatible // indirect
 	github.com/gorilla/csrf v1.0.3-0.20161122164500-69581736821c
 	github.com/gorilla/mux v1.6.1
@@ -18,20 +19,19 @@ require (
 	github.com/mattn/go-sqlite3 v1.6.0
 	github.com/ojii/gettext.go v0.0.0-20170120061437-b6dae1d7af8a
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
-	github.com/pkg/errors v0.8.1-0.20180311214515-816c9085562c
+	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v1.1.0
 	github.com/snapcore/bolt v1.3.1 // indirect
 	github.com/snapcore/go-gettext v0.0.0-20191107141714-82bbea49e785 // indirect
 	github.com/snapcore/snapd v0.0.0-20200317200833-16631e228c07
 	github.com/yohcop/openid-go v0.0.0-20170901155220-cfc72ed89575
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
-	golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
+	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
+	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
+	golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a
 	gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
 	gopkg.in/errgo.v1 v1.0.0-20161222125816-442357a80af5
 	gopkg.in/macaroon.v1 v1.0.0-20170816141150-ab101776739e
 	gopkg.in/retry.v1 v1.0.0
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v2 v2.2.4
 )

go.sum

@@ -20,7 +20,9 @@
 package service
 
 import (
+	"context"
 	"encoding/json"
+	"runtime/debug"
 
 	"net/http"
 	"os"
@@ -29,6 +31,8 @@ import (
 	"github.com/CanonicalLtd/serial-vault/datastore"
 	"github.com/CanonicalLtd/serial-vault/service/log"
 	"github.com/CanonicalLtd/serial-vault/service/response"
+	servicesentry "github.com/CanonicalLtd/serial-vault/service/sentry"
+	"github.com/getsentry/sentry-go"
 	"github.com/gorilla/csrf"
 )
 
@@ -38,14 +42,18 @@ func Logger(start time.Time, r *http.Request) {
 }
 
 // ErrorHandler is a standard error handler middleware that generates the error response
+// and sentry reports
 func ErrorHandler(f func(http.ResponseWriter, *http.Request) response.ErrorResponse) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
 		// Call the handler and it will return a custom error
 		e := f(w, r)
 		if !e.Success {
 			w.Header().Set("Content-Type", "application/json; charset=UTF-8")
 			w.WriteHeader(e.StatusCode)
 
+			servicesentry.Report(ctx, e)
+
 			// Encode the response as JSON
 			if err := json.NewEncoder(w).Encode(e); err != nil {
 				log.Printf("Error forming the signing response: %v\n", err)
@@ -58,11 +66,17 @@ func ErrorHandler(f func(http.ResponseWriter, *http.Request) response.ErrorRespo
 func Middleware(inner http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
-
+		ctx := r.Context()
+		hub := sentry.CurrentHub().Clone()
+		// add the current request to the sentry scope
+		// it will be automatically added to the report
+		hub.Scope().SetRequest(r)
+		ctx = sentry.SetHubOnContext(ctx, hub)
+		defer recoverWithSentry(hub, w, r)
 		// Log the request
 		Logger(start, r)
 
-		inner.ServeHTTP(w, r)
+		inner.ServeHTTP(w, r.WithContext(ctx))
 	})
 }
 
@@ -83,3 +97,24 @@ var MiddlewareWithCSRF = func(inner http.Handler) http.Handler {
 
 	return CSRF(Middleware(inner))
 }
+
+func recoverWithSentry(hub *sentry.Hub, w http.ResponseWriter, r *http.Request) {
+	if err := recover(); err != nil {
+		log.Errorf("recover from panic: %v", err)
+		debug.PrintStack()
+
+		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
+		e := response.ErrorInternal
+		w.WriteHeader(e.StatusCode)
+
+		// Encode the response as JSON
+		if err := json.NewEncoder(w).Encode(e); err != nil {
+			log.Printf("Error forming the error response after recovering from panic: %v\n", err)
+		}
+
+		hub.RecoverWithContext(
+			context.WithValue(r.Context(), sentry.RequestContextKey, r),
+			err,
+		)
+	}
+}

service/middleware.go

@@ -65,4 +65,5 @@ var (
 	ErrorAccountAssertion          = ErrorResponse{false, "account-assertion", "", "Error retrieving the account assertion from the database", http.StatusBadRequest}
 	ErrorSignAssertion             = ErrorResponse{false, "signing-assertion", "", "Error signing the assertion", http.StatusBadRequest}
 	ErrorGenerateNonce             = ErrorResponse{false, "generate-nonce", "", "Error generating a nonce. Please try again later", http.StatusBadRequest}
+	ErrorInternal                  = ErrorResponse{false, "server-error", "", "Internal Server Error", http.StatusInternalServerError}
 )

service/response/errors.go

@@ -0,0 +1,117 @@
+package sentry
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/CanonicalLtd/serial-vault/service/response"
+	"github.com/getsentry/sentry-go"
+)
+
+const placeholder = "[Filtered]"
+
+var headerFilter = []string{"Api-Key", "Authorization", "User", "Cookie"}
+
+// Init initializes sentry client
+func Init(dsn, serviceName, serviceVersion string) error {
+	sentrySyncTransport := sentry.NewHTTPSyncTransport()
+	sentrySyncTransport.Timeout = time.Second * 3
+
+	opt := sentry.ClientOptions{
+		Dsn:              dsn,
+		Release:          serviceVersion,
+		Transport:        sentrySyncTransport,
+		AttachStacktrace: true,
+		BeforeSend:       getFilteredEvent,
+	}
+	if err := sentry.Init(opt); err != nil {
+		return err
+	}
+
+	sentry.ConfigureScope(func(scope *sentry.Scope) {
+		scope.SetTag("app_name", serviceName)
+	})
+	return nil
+}
+
+// Report creates sentry report from response.ErrorResponse
+func Report(ctx context.Context, resp response.ErrorResponse) {
+	if !shouldReport(resp) {
+		return
+	}
+
+	if hub := sentry.GetHubFromContext(ctx); hub != nil {
+		go hub.WithScope(func(scope *sentry.Scope) {
+			// create a sentry report
+			scope.SetTag("http_status_code", fmt.Sprint(resp.StatusCode))
+			scope.SetTag("error_code", resp.Code)
+			if resp.SubCode != "" {
+				scope.SetTag("error_subcode", resp.SubCode)
+			}
+			scope.SetLevel(sentry.LevelError)
+			hub.CaptureMessage(resp.Message)
+		})
+	}
+}
+
+func getFilteredEvent(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
+	for _, headerName := range headerFilter {
+		if _, ok := event.Request.Headers[headerName]; ok {
+			event.Request.Headers[headerName] = placeholder
+		}
+	}
+
+	if event.Request.QueryString != "" {
+		event.Request.QueryString = GetFilteredQueryString(event.Request.QueryString)
+	}
+
+	if event.Request.Data != "" {
+		event.Request.Data = placeholder
+	}
+
+	if event.Request.Cookies != "" {
+		event.Request.Cookies = GetFilteredCookies(event.Request.Cookies)
+	}
+
+	return event
+}
+
+func shouldReport(resp response.ErrorResponse) bool {
+	return resp.StatusCode >= 500
+}
+
+// GetFilteredQueryString replaces all the values in the query string
+func GetFilteredQueryString(q string) string {
+	values, err := url.ParseQuery(q)
+	if err != nil {
+		return placeholder
+	}
+
+	for name := range values {
+		values.Set(name, placeholder)
+	}
+
+	return values.Encode()
+}
+
+// GetFilteredCookies replaces all the values in the cookie string
+func GetFilteredCookies(rawCookies string) string {
+	header := http.Header{}
+	header.Add("Cookie", rawCookies)
+	request := &http.Request{Header: header}
+
+	var newCookies string
+	for _, c := range request.Cookies() {
+		s := fmt.Sprintf("%s=%s", c.Name, placeholder)
+		if newCookies != "" {
+			newCookies = newCookies + "; " + s
+		} else {
+			newCookies = s
+		}
+
+	}
+	return newCookies
+}