cmd/evm, eth/tracers: refactor structlogger and make it streaming (#30806)

This PR refactors the structlog a bit, making it so that it can be used
in a streaming mode.

-------------

OBS: this PR makes a change in the input `config` config, the third
input-parem field to `debug.traceCall`. Previously, seteting it to e.g.
` {"enableMemory": true, "limit": 1024}` would mean that the response
was limited to `1024` items. Since an 'item' may include both memory and
storage, the actual size of the response was undertermined.
After this change, the response will be limited to `1024` __`bytes`__
(or thereabouts).



-----------


The commandline usage of structlog now uses the streaming mode, leaving
the non-streaming mode of operation for the eth_Call.

There are two benefits of streaming mode 
1. Not have to maintain a long list of operations, 
2. Not have to duplicate / n-plicate data, e.g. memory / stack /
returndata so that each entry has their own private slice.


---------

Co-authored-by: Gary Rong <[email protected]>

Author: holiman

cmd/evm/main.go

@@ -240,13 +240,13 @@ func tracerFromFlags(ctx *cli.Context) *tracing.Hooks {
 	}
 	switch {
 	case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "struct":
-		return logger.NewStructLogger(config).Hooks()
+		return logger.NewStreamingStructLogger(config, os.Stderr).Hooks()
 	case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "json":
 		return logger.NewJSONLogger(config, os.Stderr)
 	case ctx.Bool(MachineFlag.Name):
 		return logger.NewJSONLogger(config, os.Stderr)
 	case ctx.Bool(DebugFlag.Name):
-		return logger.NewStructLogger(config).Hooks()
+		return logger.NewStreamingStructLogger(config, os.Stderr).Hooks()
 	default:
 		return nil
 	}

cmd/evm/runner.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -35,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/tracing"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
@@ -205,7 +207,6 @@ func runCmd(ctx *cli.Context) error {
 
 	var (
 		tracer      *tracing.Hooks
-		debugLogger *logger.StructLogger
 		prestate    *state.StateDB
 		chainConfig *params.ChainConfig
 		sender      = common.BytesToAddress([]byte("sender"))
@@ -217,10 +218,7 @@ func runCmd(ctx *cli.Context) error {
 	if ctx.Bool(MachineFlag.Name) {
 		tracer = logger.NewJSONLogger(logconfig, os.Stdout)
 	} else if ctx.Bool(DebugFlag.Name) {
-		debugLogger = logger.NewStructLogger(logconfig)
-		tracer = debugLogger.Hooks()
-	} else {
-		debugLogger = logger.NewStructLogger(logconfig)
+		tracer = logger.NewStreamingStructLogger(logconfig, os.Stderr).Hooks()
 	}
 
 	initialGas := ctx.Uint64(GasFlag.Name)
@@ -365,12 +363,10 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	if ctx.Bool(DebugFlag.Name) {
-		if debugLogger != nil {
-			fmt.Fprintln(os.Stderr, "#### TRACE ####")
-			logger.WriteTrace(os.Stderr, debugLogger.StructLogs())
+		if logs := runtimeConfig.State.Logs(); len(logs) > 0 {
+			fmt.Fprintln(os.Stderr, "### LOGS")
+			writeLogs(os.Stderr, logs)
 		}
-		fmt.Fprintln(os.Stderr, "#### LOGS ####")
-		logger.WriteLogs(os.Stderr, runtimeConfig.State.Logs())
 	}
 
 	if bench || ctx.Bool(StatDumpFlag.Name) {
@@ -389,3 +385,16 @@ allocated bytes: %d
 
 	return nil
 }
+
+// writeLogs writes vm logs in a readable format to the given writer
+func writeLogs(writer io.Writer, logs []*types.Log) {
+	for _, log := range logs {
+		fmt.Fprintf(writer, "LOG%d: %x bn=%d txi=%x\n", len(log.Topics), log.Address, log.BlockNumber, log.TxIndex)
+
+		for i, topic := range log.Topics {
+			fmt.Fprintf(writer, "%08d  %x\n", i, topic)
+		}
+		fmt.Fprint(writer, hex.Dump(log.Data))
+		fmt.Fprintln(writer)
+	}
+}

core/vm/runtime/runtime_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/asm"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/program"
@@ -670,17 +671,23 @@ func TestColdAccountAccessCost(t *testing.T) {
 			want: 7600,
 		},
 	} {
-		tracer := logger.NewStructLogger(nil)
+		var step = 0
+		var have = uint64(0)
 		Execute(tc.code, nil, &Config{
 			EVMConfig: vm.Config{
-				Tracer: tracer.Hooks(),
+				Tracer: &tracing.Hooks{
+					OnOpcode: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) {
+						// Uncomment to investigate failures:
+						//t.Logf("%d: %v %d", step, vm.OpCode(op).String(), cost)
+						if step == tc.step {
+							have = cost
+						}
+						step++
+					},
+				},
 			},
 		})
-		have := tracer.StructLogs()[tc.step].GasCost
 		if want := tc.want; have != want {
-			for ii, op := range tracer.StructLogs() {
-				t.Logf("%d: %v %d", ii, op.OpName(), op.GasCost)
-			}
 			t.Fatalf("testcase %d, gas report wrong, step %d, have %d want %d", i, tc.step, have, want)
 		}
 	}

eth/tracers/api_test.go

@@ -535,7 +535,7 @@ func TestTraceTransaction(t *testing.T) {
 		Gas:         params.TxGas,
 		Failed:      false,
 		ReturnValue: "",
-		StructLogs:  []logger.StructLogRes{},
+		StructLogs:  []json.RawMessage{},
 	}) {
 		t.Error("Transaction tracing result is different")
 	}