add support for tracing

Author: turbolent

cmd/util/cmd/debug-script/cmd.go

@@ -127,7 +127,9 @@ func run(*cobra.Command, []string) {
 		}
 	}
 
-	debugger := debug.NewRemoteDebugger(chain, log.Logger)
+	// TODO:
+	const traceCadence = false
+	debugger := debug.NewRemoteDebugger(chain, log.Logger, traceCadence)
 
 	// TODO: add support for arguments
 	var arguments [][]byte

cmd/util/cmd/debug-tx/cmd.go

@@ -13,15 +13,18 @@ import (
 	"github.com/onflow/flow/protobuf/go/flow/executiondata"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
 	"golang.org/x/exp/slices"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 
 	sdk "github.com/onflow/flow-go-sdk"
 
+	"github.com/onflow/flow-go/fvm"
 	"github.com/onflow/flow-go/fvm/storage/snapshot"
 	"github.com/onflow/flow-go/model/flow"
 	"github.com/onflow/flow-go/module/grpcclient"
+	"github.com/onflow/flow-go/module/trace"
 	"github.com/onflow/flow-go/utils/debug"
 )
 
@@ -36,6 +39,8 @@ var (
 	flagProposalKeySeq      uint64
 	flagUseExecutionDataAPI bool
 	flagDumpRegisters       bool
+	flagTracePath           string
+	flagTraceCadence        bool
 )
 
 var Cmd = &cobra.Command{
@@ -67,6 +72,10 @@ func init() {
 	Cmd.Flags().BoolVar(&flagUseExecutionDataAPI, "use-execution-data-api", false, "use the execution data API")
 
 	Cmd.Flags().BoolVar(&flagDumpRegisters, "dump-registers", false, "dump registers")
+
+	Cmd.Flags().StringVar(&flagTracePath, "trace", "", "enable tracing to given path")
+
+	Cmd.Flags().BoolVar(&flagTraceCadence, "trace-cadence", false, "trace Cadence")
 }
 
 func run(_ *cobra.Command, args []string) {
@@ -178,7 +187,55 @@ func runTransactionID(txID flow.Identifier, flowClient *client.Client, chain flo
 
 	blockSnapshot := newBlockSnapshot(remoteSnapshot)
 
-	debugger := debug.NewRemoteDebugger(chain, log.Logger)
+	var fvmOptions []fvm.Option
+
+	if flagTracePath != "" {
+
+		var traceFile *os.File
+		if flagTracePath == "-" {
+			traceFile = os.Stdout
+		} else {
+			traceFile, err = os.Create(flagTracePath)
+			if err != nil {
+				log.Fatal().Err(err).Msg("failed to create trace file")
+			}
+			defer traceFile.Close()
+		}
+
+		exporter, err := stdouttrace.New(
+			stdouttrace.WithWriter(traceFile),
+		)
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed to create trace exporter")
+		}
+
+		tracer, err := trace.NewTracerWithExporter(
+			log.Logger,
+			"debug-tx",
+			flagChain,
+			trace.SensitivityCaptureAll,
+			exporter,
+		)
+		if err != nil {
+			log.Fatal().Err(err).Msg("failed to create tracer")
+		}
+
+		span, _ := tracer.StartTransactionSpan(context.TODO(), txID, "")
+		defer span.End()
+
+		fvmOptions = append(
+			fvmOptions,
+			fvm.WithTracer(tracer),
+			fvm.WithSpan(span),
+		)
+	}
+
+	debugger := debug.NewRemoteDebugger(
+		chain,
+		log.Logger,
+		flagTraceCadence,
+		fvmOptions...,
+	)
 
 	for _, blockTx := range txsResult {
 		blockTxID := flow.Identifier(blockTx.ID())

go.mod

@@ -112,6 +112,7 @@ require (
 	github.com/onflow/wal v1.0.2
 	github.com/slok/go-http-metrics v0.12.0
 	github.com/sony/gobreaker v0.5.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0
 	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da
 	google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a
 	google.golang.org/genproto/googleapis/bytestream v0.0.0-20250603155806-513f23925822

go.sum

@@ -1354,6 +1354,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0 h1:FFeLy
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.31.0/go.mod h1:TMu73/k1CP8nBUpDLc71Wj/Kf7ZS9FK5b53VapRsP9o=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc=
 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 h1:VhlEQAPp9R1ktYfrPk5SOryw1e9LDDTZCbIPFrho0ec=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0/go.mod h1:kB3ufRbfU+CQ4MlUcqtW8Z7YEOBeK2DJ6CmR5rYYF3E=
 go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
 go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
 go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=

module/trace/noop.go

@@ -63,6 +63,18 @@ func (t *NoopTracer) StartCollectionSpan(
 	return NoopSpan, ctx
 }
 
+func (t *NoopTracer) StartTransactionSpan(
+	ctx context.Context,
+	entityID flow.Identifier,
+	spanName SpanName,
+	opts ...trace.SpanStartOption,
+) (
+	trace.Span,
+	context.Context,
+) {
+	return NoopSpan, ctx
+}
+
 func (t *NoopTracer) StartSpanFromContext(
 	ctx context.Context,
 	operationName SpanName,

module/trace/trace.go

@@ -55,6 +55,36 @@ func NewTracer(
 ) (
 	*Tracer,
 	error,
+) {
+	ctx := context.TODO()
+	// OLTP trace gRPC client initialization. Connection parameters for the exporter are extracted
+	// from environment variables. e.g.: `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`.
+	//
+	// For more information, see OpenTelemetry specification:
+	// https://github.com/open-telemetry/opentelemetry-specification/blob/v1.12.0/specification/protocol/exporter.md
+	traceExporter, err := otlptracegrpc.New(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create trace exporter: %w", err)
+	}
+
+	return NewTracerWithExporter(
+		log,
+		serviceName,
+		chainID,
+		sensitivity,
+		traceExporter,
+	)
+}
+
+func NewTracerWithExporter(
+	log zerolog.Logger,
+	serviceName string,
+	chainID string,
+	sensitivity uint,
+	traceExporter sdktrace.SpanExporter,
+) (
+	*Tracer,
+	error,
 ) {
 	ctx := context.TODO()
 	res, err := resource.New(
@@ -68,16 +98,6 @@ func NewTracer(
 		return nil, fmt.Errorf("failed to create resource: %w", err)
 	}
 
-	// OLTP trace gRPC client initialization. Connection parameters for the exporter are extracted
-	// from environment variables. e.g.: `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`.
-	//
-	// For more information, see OpenTelemetry specification:
-	// https://github.com/open-telemetry/opentelemetry-specification/blob/v1.12.0/specification/protocol/exporter.md
-	traceExporter, err := otlptracegrpc.New(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create trace exporter: %w", err)
-	}
-
 	tracerProvider := sdktrace.NewTracerProvider(
 		sdktrace.WithResource(res),
 		sdktrace.WithBatcher(traceExporter),
@@ -207,6 +227,18 @@ func (t *Tracer) StartCollectionSpan(
 	return t.startEntitySpan(ctx, collectionID, EntityTypeCollection, spanName, opts...)
 }
 
+func (t *Tracer) StartTransactionSpan(
+	ctx context.Context,
+	transactionID flow.Identifier,
+	spanName SpanName,
+	opts ...trace.SpanStartOption,
+) (
+	trace.Span,
+	context.Context,
+) {
+	return t.startEntitySpan(ctx, transactionID, EntityTypeTransaction, spanName, opts...)
+}
+
 func (t *Tracer) StartSpanFromContext(
 	ctx context.Context,
 	operationName SpanName,

module/tracer.go

@@ -35,7 +35,7 @@ type Tracer interface {
 		context.Context,
 	)
 
-	// StartCollectionSpan starts an span for a collection, built as a child of
+	// StartCollectionSpan starts a span for a collection, built as a child of
 	// rootSpan.  It also returns the context including this span which can be
 	// used for nested calls.
 	StartCollectionSpan(
@@ -48,6 +48,16 @@ type Tracer interface {
 		context.Context,
 	)
 
+	StartTransactionSpan(
+		ctx context.Context,
+		transactionID flow.Identifier,
+		spanName trace.SpanName,
+		opts ...otelTrace.SpanStartOption,
+	) (
+		otelTrace.Span,
+		context.Context,
+	)
+
 	StartSpanFromContext(
 		ctx context.Context,
 		operationName trace.SpanName,

utils/debug/remoteDebugger.go

@@ -2,9 +2,11 @@ package debug
 
 import (
 	"github.com/onflow/cadence"
+	"github.com/onflow/cadence/runtime"
 	"github.com/rs/zerolog"
 
 	"github.com/onflow/flow-go/fvm"
+	reusableRuntime "github.com/onflow/flow-go/fvm/runtime"
 	"github.com/onflow/flow-go/fvm/storage/snapshot"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -14,22 +16,42 @@ type RemoteDebugger struct {
 	ctx fvm.Context
 }
 
+// ReusableCadenceRuntimePoolSize is the size of the reusable cadence runtime pool.
+// Copied from engine/execution/computation/manager.go to avoid circular dependency.
+const reusableCadenceRuntimePoolSize = 1000
+
 // NewRemoteDebugger creates a new remote debugger.
 // NOTE: Make sure to use the same version of flow-go as the network
 // you are collecting registers from, otherwise the execution might differ
 // from the way it runs on the network
 func NewRemoteDebugger(
 	chain flow.Chain,
 	logger zerolog.Logger,
+	traceCadence bool,
+	options ...fvm.Option,
 ) *RemoteDebugger {
 	vm := fvm.NewVirtualMachine()
 
 	// no signature processor here
 	// TODO Maybe we add fee-deduction step as well
+
 	ctx := fvm.NewContext(
-		fvm.WithLogger(logger),
-		fvm.WithChain(chain),
-		fvm.WithAuthorizationChecksEnabled(false),
+		append(
+			[]fvm.Option{
+				fvm.WithLogger(logger),
+				fvm.WithChain(chain),
+				fvm.WithAuthorizationChecksEnabled(false),
+				fvm.WithReusableCadenceRuntimePool(
+					reusableRuntime.NewReusableCadenceRuntimePool(
+						reusableCadenceRuntimePoolSize,
+						runtime.Config{
+							TracingEnabled: traceCadence,
+						},
+					)),
+				fvm.WithEVMEnabled(true),
+			},
+			options...,
+		)...,
 	)
 
 	return &RemoteDebugger{