Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ require (
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d
github.com/bsm/redislock v0.9.3
github.com/caarlos0/env/v11 v11.3.1
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d
github.com/dave/jennifer v1.4.1
github.com/getkin/kin-openapi v0.129.0
github.com/getsentry/sentry-go v0.31.1
Expand All @@ -23,10 +22,9 @@ require (
github.com/jpillora/backoff v1.0.0
github.com/mattn/go-isatty v0.0.20
github.com/minio/minio-go/v7 v7.0.87
github.com/pkg/errors v0.9.1
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
github.com/prometheus/client_golang v1.21.1
github.com/redis/go-redis/v9 v9.7.1
github.com/redis/go-redis/v9 v9.7.3
github.com/rs/xid v1.6.0
github.com/rs/zerolog v1.33.0
github.com/shopspring/decimal v1.4.0
Expand Down
60 changes: 5 additions & 55 deletions go.sum

Large diffs are not rendered by default.

55 changes: 55 additions & 0 deletions internal/sentry/sentry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package sentry

import (
"log"
"os"
"strconv"
"strings"

"github.com/getsentry/sentry-go"
)

func init() {
var (
tracesSampleRate float64 = 0.1
enableTracing = true
)

val := strings.TrimSpace(os.Getenv("SENTRY_TRACES_SAMPLE_RATE"))
if val != "" {
var err error

tracesSampleRate, err = strconv.ParseFloat(val, 64)
if err != nil {
log.Fatalf("failed to parse SENTRY_TRACES_SAMPLE_RATE: %v", err)
}
}

valEnableTracing := strings.TrimSpace(os.Getenv("SENTRY_ENABLE_TRACING"))
if valEnableTracing != "" {
var err error

enableTracing, err = strconv.ParseBool(valEnableTracing)
if err != nil {
log.Fatalf("failed to parse SENTRY_ENABLE_TRACING: %v", err)
}
}

err := sentry.Init(sentry.ClientOptions{
Dsn: os.Getenv("SENTRY_DSN"),
Environment: os.Getenv("ENVIRONMENT"),
EnableTracing: enableTracing,
TracesSampleRate: tracesSampleRate,
BeforeSendTransaction: func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event {
// Drop request body.
if event.Request != nil {
event.Request.Data = ""
}

return event
},
})
if err != nil {
log.Fatalf("sentry.Init: %v", err)
}
}
210 changes: 105 additions & 105 deletions maintenance/errors/error.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,26 +11,54 @@ import (
"os"
"time"

"github.com/getsentry/sentry-go"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog"

"github.com/pace/bricks/http/jsonapi/runtime"
"github.com/pace/bricks/http/oauth2"
"github.com/pace/bricks/maintenance/errors/raven"
_ "github.com/pace/bricks/internal/sentry"
"github.com/pace/bricks/maintenance/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog"
)

var paceHTTPPanicCounter = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "pace_http_panic_total",
Help: "A counter for panics intercepted while handling a request",
})

var DefaultClient *sentry.Client

func init() {
prometheus.MustRegister(paceHTTPPanicCounter)
}

// PanicWrap wraps a panic for HandleRequest
type PanicWrap struct {
err interface{}
type ErrWithExtra struct {
err error
extra map[string]any
}

func NewErrWithExtra(err error, extra map[string]any) ErrWithExtra {
return ErrWithExtra{
err: err,
extra: extra,
}
}

func (e ErrWithExtra) Error() string {
return e.err.Error()
}

// Panic wraps a panic for HandleRequest
type Panic struct {
err any
}

func NewPanic(err any) Panic {
return Panic{err: err}
}

func (p Panic) Error() string {
return fmt.Sprintf("%v", p.err)
}

type recoveryHandler struct {
Expand Down Expand Up @@ -88,153 +116,125 @@ func Handler() func(http.Handler) http.Handler {

// HandleRequest should be called with defer to recover panics in request handlers
func HandleRequest(handlerName string, w http.ResponseWriter, r *http.Request) {
if rp := recover(); rp != nil {
if rec := recover(); rec != nil {
paceHTTPPanicCounter.Inc()
HandleError(&PanicWrap{rp}, handlerName, w, r)
HandleError(NewPanic(rec), handlerName, w, r)
}
}

// HandleError reports the passed error to sentry
func HandleError(rp interface{}, handlerName string, w http.ResponseWriter, r *http.Request) {
func HandleError(err error, handlerName string, w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
pw, ok := rp.(*PanicWrap)
if ok {
log.Ctx(ctx).Error().Str("handler", handlerName).Msgf("Panic: %v", pw.err)
rp = pw.err // unwrap error
} else {
log.Ctx(ctx).Error().Str("handler", handlerName).Msgf("Error: %v", rp)
}
log.Stack(ctx)

sentryEvent{ctx, r, rp, 1, handlerName}.Send()
handle(ctx, err, handlerName)

runtime.WriteError(w, http.StatusInternalServerError, errors.New("Internal Server Error"))
runtime.WriteError(w, http.StatusInternalServerError, errors.New("internal Server Error"))
}

// Handle logs the given error and reports it to sentry.
func Handle(ctx context.Context, rp interface{}) {
pw, ok := rp.(*PanicWrap)
if ok {
log.Ctx(ctx).Error().Msgf("Panic: %v", pw.err)
rp = pw.err // unwrap error
} else {
log.Ctx(ctx).Error().Msgf("Error: %v", rp)
}
log.Stack(ctx)

sentryEvent{ctx, nil, rp, 1, ""}.Send()
func Handle(ctx context.Context, err error) {
handle(ctx, err, "")
}

// HandleWithCtx should be called with defer to recover panics in goroutines
func HandleWithCtx(ctx context.Context, handlerName string) {
if rp := recover(); rp != nil {
log.Ctx(ctx).Error().Str("handler", handlerName).Msgf("Panic: %v", rp)
log.Stack(ctx)
func handle(ctx context.Context, err error, handlerName string) {
l := log.Ctx(ctx).Error().Err(err)

sentryEvent{ctx, nil, rp, 2, handlerName}.Send()
if handlerName != "" {
l = l.Str("handler", handlerName)
}
}

func HandleErrorNoStack(ctx context.Context, err error) {
log.Ctx(ctx).Info().Msgf("Received error, will not handle further: %v", err)
}

// New returns an error that formats as the given text.
func New(text string) error {
return errors.New(text)
}
var p Panic

// WrapWithExtra adds extra data to an error before reporting to Sentry
func WrapWithExtra(err error, extraInfo map[string]interface{}) error {
return raven.WrapWithExtra(err, extraInfo)
}
if errors.As(err, &p) {
l.Msg("Panic")
} else {
l.Msg("Error")
}

type sentryEvent struct {
ctx context.Context
req *http.Request // optional
r interface{}
level int
handlerName string
}
log.Stack(ctx)

func (e sentryEvent) Send() {
_, errCh := raven.Capture(e.build(), nil)
<-errCh // ensure the message get send even if the main goroutine is about to stop
sentry.CaptureEvent(getEvent(ctx, nil, err, 1, handlerName))
}

func (e sentryEvent) build() *raven.Packet {
ctx, r, rp, handlerName := e.ctx, e.req, e.r, e.handlerName

func getEvent(ctx context.Context, r *http.Request, err error, level int, handlerName string) *sentry.Event {
// get request from context if available
if r == nil {
r = requestFromContext(ctx)
}

rvalStr := fmt.Sprint(rp)
var packet *raven.Packet

if err, ok := rp.(error); ok {
stack := raven.GetOrNewStacktrace(err, 2+e.level, 3, nil)
packet = raven.NewPacket(rvalStr, raven.NewException(err, stack))
} else {
stack := raven.NewStacktrace(2+e.level, 3, nil)
packet = raven.NewPacket(rvalStr, raven.NewException(errors.New(rvalStr), stack))
}
event := sentry.NewEvent()

// extract ErrWithExtra info and append it to the packet
if ee, ok := rp.(raven.ErrWithExtra); ok {
for k, v := range ee.ExtraInfo() {
packet.Extra[k] = v
}
}
event.SetException(err, 2+level)

// add user
userID, ok := oauth2.UserID(ctx)
user := raven.User{ID: userID}
if r != nil {
user.IP = log.ProxyAwareRemote(r)
}
packet.Interfaces = append(packet.Interfaces, &user)
if ok {
packet.Tags = append(packet.Tags, raven.Tag{Key: "user_id", Value: userID})
event.User.ID = userID
}

if r != nil {
event.User.IPAddress = log.ProxyAwareRemote(r)
}

// from context
if reqID := log.RequestIDFromContext(ctx); reqID != "" {
packet.Extra["req_id"] = reqID
packet.Tags = append(packet.Tags, raven.Tag{Key: "req_id", Value: reqID})
event.Extra["req_id"] = reqID
event.Tags["req_id"] = reqID
}

if traceID := log.TraceIDFromContext(ctx); traceID != "" {
packet.Extra["uber_trace_id"] = traceID
packet.Tags = append(packet.Tags, raven.Tag{Key: "trace_id", Value: traceID})
event.Extra["uber_trace_id"] = traceID
event.Tags["trace_id"] = traceID
}
packet.Extra["handler"] = handlerName

event.Extra["handler"] = handlerName

if clientID, ok := oauth2.ClientID(ctx); ok {
packet.Extra["oauth2_client_id"] = clientID
}
if scopes := oauth2.Scopes(ctx); len(scopes) > 0 {
packet.Extra["oauth2_scopes"] = scopes
event.Extra["oauth2_client_id"] = clientID
}

// from request
if r != nil {
packet.Interfaces = append(packet.Interfaces, raven.NewHttp(r))
if scopes := oauth2.Scopes(ctx); len(scopes) > 0 {
event.Extra["oauth2_scopes"] = scopes
}

// from env
packet.Extra["microservice"] = os.Getenv("JAEGER_SERVICE_NAME")
event.Extra["microservice"] = os.Getenv("JAEGER_SERVICE_NAME")

// add breadcrumbs
packet.Breadcrumbs = getBreadcrumbs(ctx)
event.Breadcrumbs = getBreadcrumbs(ctx)

return event
}

// HandleWithCtx should be called with defer to recover panics in goroutines
func HandleWithCtx(ctx context.Context, handlerName string) {
if r := recover(); r != nil {
log.Ctx(ctx).Error().Str("handler", handlerName).Msgf("Panic: %v", r)
log.Stack(ctx)

sentry.CaptureEvent(getEvent(ctx, nil, NewPanic(r), 2, handlerName))
}
}

return packet
func HandleErrorNoStack(ctx context.Context, err error) {
log.Ctx(ctx).Info().Msgf("Received error, will not handle further: %v", err)
}

// New returns an error that formats as the given text.
func New(text string) error {
return errors.New(text)
}

// WrapWithExtra adds extra data to an error before reporting to Sentry
func WrapWithExtra(err error, extraInfo map[string]any) error {
return NewErrWithExtra(err, extraInfo)
}

// getBreadcrumbs takes a context and tries to extract the logs from it if it
// holds a log.Sink. If that's the case, the logs will all be translated
// to valid sentry breadcrumbs if possible. In case of a failure, the
// breadcrumbs will be dropped and a warning will be logged.
func getBreadcrumbs(ctx context.Context) []*raven.Breadcrumb {
func getBreadcrumbs(ctx context.Context) []*sentry.Breadcrumb {
sink, ok := log.SinkFromContext(ctx)
if !ok {
return nil
Expand All @@ -246,7 +246,7 @@ func getBreadcrumbs(ctx context.Context) []*raven.Breadcrumb {
return nil
}

result := make([]*raven.Breadcrumb, len(data))
result := make([]*sentry.Breadcrumb, len(data))
for i, d := range data {
crumb, err := createBreadcrumb(d)
if err != nil {
Expand All @@ -260,7 +260,7 @@ func getBreadcrumbs(ctx context.Context) []*raven.Breadcrumb {
return result
}

func createBreadcrumb(data map[string]interface{}) (*raven.Breadcrumb, error) {
func createBreadcrumb(data map[string]any) (*sentry.Breadcrumb, error) {
// remove the request id if it can still be found in the logs
delete(data, "req_id")

Expand Down Expand Up @@ -318,11 +318,11 @@ func createBreadcrumb(data map[string]interface{}) (*raven.Breadcrumb, error) {
typ = "error"
}

return &raven.Breadcrumb{
return &sentry.Breadcrumb{
Category: category,
Level: level,
Message: message,
Timestamp: time.Unix(),
Timestamp: time,
Type: typ,
Data: data,
}, nil
Expand All @@ -331,7 +331,7 @@ func createBreadcrumb(data map[string]interface{}) (*raven.Breadcrumb, error) {
// translateZerologLevelToSentryLevel takes in a zerolog.Level as string
// and returns the equivalent sentry breadcrumb level. If the given level
// can't be parsed to a valid zerolog.Level an error is returned.
func translateZerologLevelToSentryLevel(l string) (string, error) {
func translateZerologLevelToSentryLevel(l string) (sentry.Level, error) {
level, err := zerolog.ParseLevel(l)
if err != nil {
return "", err
Expand Down
Loading