diff --git a/cmd/opera/launcher/launcher.go b/cmd/opera/launcher/launcher.go index 59c848136..f08be6e8b 100644 --- a/cmd/opera/launcher/launcher.go +++ b/cmd/opera/launcher/launcher.go @@ -54,6 +54,7 @@ var ( legacyRpcFlags []cli.Flag rpcFlags []cli.Flag metricsFlags []cli.Flag + tracingFlags []cli.Flag ) func initFlags() { @@ -157,7 +158,12 @@ func initFlags() { utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + } + + tracingFlags = []cli.Flag{ tracing.EnableFlag, + tracing.AgentEndpointFlag, + tracing.EnableDevFlag, } nodeFlags = []cli.Flag{} @@ -213,6 +219,7 @@ func init() { app.Flags = append(app.Flags, consoleFlags...) app.Flags = append(app.Flags, debug.Flags...) app.Flags = append(app.Flags, metricsFlags...) + app.Flags = append(app.Flags, tracingFlags...) app.Before = func(ctx *cli.Context) error { if err := debug.Setup(ctx); err != nil { @@ -246,12 +253,11 @@ func lachesisMain(ctx *cli.Context) error { return fmt.Errorf("invalid command: %q", args[0]) } - // TODO: tracing flags - //tracingStop, err := tracing.Start(ctx) - //if err != nil { - // return err - //} - //defer tracingStop() + tracingStop, err := tracing.Start(ctx) + if err != nil { + return err + } + defer tracingStop() cfg := makeAllConfigs(ctx) genesisPath := getOperaGenesis(ctx) diff --git a/cmd/opera/launcher/tracing/tracing.go b/cmd/opera/launcher/tracing/tracing.go index ea8b503fe..75b7bd9b4 100644 --- a/cmd/opera/launcher/tracing/tracing.go +++ b/cmd/opera/launcher/tracing/tracing.go @@ -1,7 +1,10 @@ package tracing import ( + "errors" + "fmt" opentracing "github.com/opentracing/opentracing-go" + "github.com/uber/jaeger-client-go" jaegercfg "github.com/uber/jaeger-client-go/config" jaegerlog "github.com/uber/jaeger-client-go/log" "github.com/uber/jaeger-lib/metrics" @@ -10,38 +13,65 @@ import ( "github.com/Fantom-foundation/go-opera/tracing" ) -var EnableFlag = cli.BoolFlag{ - Name: "tracing", - Usage: "Enable traces collection and reporting", -} +var ( + EnableFlag = cli.BoolFlag{ + Name: "tracing", + Usage: "Enable traces collection and reporting", + } -func Start(ctx *cli.Context) (stop func(), err error) { - stop = func() {} + EnableDevFlag = cli.BoolFlag{ + Name: "tracing.dev", + Usage: "Enable traces collection and reporting in development mode", + } - if !ctx.Bool(EnableFlag.Name) { - return + AgentEndpointFlag = cli.StringFlag{ + Name: "tracing.agent", + Usage: "Jaeger agent endpoint. Default is localhost:6831", + Value: "localhost:6831", } - var cfg *jaegercfg.Configuration - cfg, err = jaegercfg.FromEnv() - if err != nil { - return + ErrInvalidEndpoint = errors.New("invalid agent endpoint") +) + +func Start(ctx *cli.Context) (func(), error) { + if !ctx.Bool(EnableFlag.Name) && !ctx.Bool(EnableDevFlag.Name) { + return func() {}, nil } - cfg.ServiceName = "opera" + agentEndpoint := ctx.String(AgentEndpointFlag.Name) + if agentEndpoint == "" { + return nil, ErrInvalidEndpoint + } + + // Default config recommended for production + cfg := jaegercfg.Configuration{ + ServiceName: "opera", + Reporter: &jaegercfg.ReporterConfig{ + LocalAgentHostPort: agentEndpoint, + }, + } + + if ctx.Bool(EnableDevFlag.Name) { + // Makes sampler collect and report all traces + cfg.Sampler = &jaegercfg.SamplerConfig{ + Type: jaeger.SamplerTypeConst, + Param: 1, + } + } tracer, closer, err := cfg.NewTracer( jaegercfg.Logger(jaegerlog.StdLogger), jaegercfg.Metrics(metrics.NullFactory), ) if err != nil { - return + return nil, fmt.Errorf("new tracer: %w", err) } - stop = func() { + stop := func() { closer.Close() } opentracing.SetGlobalTracer(tracer) tracing.SetEnabled(true) - return + + return stop, nil } diff --git a/demo/start.sh b/demo/start.sh index 4066c466e..693476ea1 100755 --- a/demo/start.sh +++ b/demo/start.sh @@ -24,6 +24,7 @@ do --fakenet=${ACC}/$N \ --port=${PORT} \ --nat extip:127.0.0.1 \ + --tracing.dev \ --http --http.addr="127.0.0.1" --http.port=${RPCP} --http.corsdomain="*" --http.api="eth,debug,net,admin,web3,personal,txpool,ftm,dag" \ --ws --ws.addr="127.0.0.1" --ws.port=${WSP} --ws.origins="*" --ws.api="eth,debug,net,admin,web3,personal,txpool,ftm,dag" \ --nousb --verbosity=3 --tracing &>> opera$i.log)& diff --git a/evmcore/tx_pool.go b/evmcore/tx_pool.go index f6c5af180..d62f640cb 100644 --- a/evmcore/tx_pool.go +++ b/evmcore/tx_pool.go @@ -18,6 +18,7 @@ package evmcore import ( "errors" + "github.com/Fantom-foundation/go-opera/tracing" "math" "math/big" "math/rand" @@ -1828,6 +1829,8 @@ func (t *txLookup) Remove(hash common.Hash) { t.lock.Lock() defer t.lock.Unlock() + defer tracing.FinishTx(hash, "txLookup.Remove()") + tx, ok := t.locals[hash] if !ok { tx, ok = t.remotes[hash] diff --git a/tracing/tx-tracing.go b/tracing/tx-tracing.go index d037e1811..d657b59d7 100644 --- a/tracing/tx-tracing.go +++ b/tracing/tx-tracing.go @@ -2,14 +2,21 @@ package tracing import ( "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/opentracing/opentracing-go" ) +// txSpan wraps opentracing span and stores additional payload +type txSpan struct { + opentracing.Span + begin time.Time +} + var ( enabled bool - txSpans = make(map[common.Hash]opentracing.Span) + txSpans = make(map[common.Hash]txSpan) txSpansMu sync.RWMutex noopSpan = opentracing.NoopTracer{}.StartSpan("") @@ -35,9 +42,13 @@ func StartTx(tx common.Hash, operation string) { return } - span := opentracing.StartSpan("lifecycle") + span := txSpan{ + Span: opentracing.StartSpan("lifecycle"), + begin: time.Now(), + } span.SetTag("txhash", tx.String()) span.SetTag("enter", operation) + txSpans[tx] = span } @@ -55,7 +66,9 @@ func FinishTx(tx common.Hash, operation string) { } span.SetTag("exit", operation) + span.SetTag("time.millis", time.Since(span.begin).Milliseconds()) span.Finish() + delete(txSpans, tx) }