cmd/evm: consolidate evm output switches (#30849)

This PR attempts to clean up some ambiguities and quirks from recent
changes to evm flag handling.

This PR mainly focuses on `evm run` subcommand, to use the same flags
for configuring tracing/output options as `statetest/blocktest` does.

Additionally, it adds quite a lot of tests for expected outputs of the
various subcommands, to avoid accidental changes.

---------

Co-authored-by: Felix Lange <[email protected]>

Author: holiman

cmd/evm/internal/t8ntool/transition.go

@@ -97,7 +97,6 @@ func Transition(ctx *cli.Context) error {
 			DisableStack:     ctx.Bool(TraceDisableStackFlag.Name),
 			EnableMemory:     ctx.Bool(TraceEnableMemoryFlag.Name),
 			EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name),
-			Debug:            true,
 		}
 		getTracer = func(txIndex int, txHash common.Hash, _ *params.ChainConfig) (*tracers.Tracer, io.WriteCloser, error) {
 			traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String())))

cmd/evm/main.go

@@ -239,10 +239,20 @@ func tracerFromFlags(ctx *cli.Context) *tracing.Hooks {
 		EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name),
 	}
 	switch {
-	case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "struct":
-		return logger.NewStreamingStructLogger(config, os.Stderr).Hooks()
-	case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "json":
-		return logger.NewJSONLogger(config, os.Stderr)
+	case ctx.Bool(TraceFlag.Name):
+		switch format := ctx.String(TraceFormatFlag.Name); format {
+		case "struct":
+			return logger.NewStreamingStructLogger(config, os.Stderr).Hooks()
+		case "json":
+			return logger.NewJSONLogger(config, os.Stderr)
+		case "md", "markdown":
+			return logger.NewMarkdownLogger(config, os.Stderr).Hooks()
+		default:
+			fmt.Fprintf(os.Stderr, "unknown trace format: %q\n", format)
+			os.Exit(1)
+			return nil
+		}
+	// Deprecated ways of configuring tracing.
 	case ctx.Bool(MachineFlag.Name):
 		return logger.NewJSONLogger(config, os.Stderr)
 	case ctx.Bool(DebugFlag.Name):

cmd/evm/runner.go

@@ -39,7 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
-	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/triedb"
@@ -66,6 +65,7 @@ var runCommand = &cli.Command{
 		SenderFlag,
 		ValueFlag,
 		StatDumpFlag,
+		DumpFlag,
 	}, traceFlags),
 }
 
@@ -197,14 +197,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, exe
 }
 
 func runCmd(ctx *cli.Context) error {
-	logconfig := &logger.Config{
-		EnableMemory:     !ctx.Bool(TraceDisableMemoryFlag.Name),
-		DisableStack:     ctx.Bool(TraceDisableStackFlag.Name),
-		DisableStorage:   ctx.Bool(TraceDisableStorageFlag.Name),
-		EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name),
-		Debug:            ctx.Bool(DebugFlag.Name),
-	}
-
 	var (
 		tracer      *tracing.Hooks
 		prestate    *state.StateDB
@@ -215,12 +207,7 @@ func runCmd(ctx *cli.Context) error {
 		blobHashes  []common.Hash  // TODO (MariusVanDerWijden) implement blob hashes in state tests
 		blobBaseFee = new(big.Int) // TODO (MariusVanDerWijden) implement blob fee in state tests
 	)
-	if ctx.Bool(MachineFlag.Name) {
-		tracer = logger.NewJSONLogger(logconfig, os.Stdout)
-	} else if ctx.Bool(DebugFlag.Name) {
-		tracer = logger.NewStreamingStructLogger(logconfig, os.Stderr).Hooks()
-	}
-
+	tracer = tracerFromFlags(ctx)
 	initialGas := ctx.Uint64(GasFlag.Name)
 	genesisConfig := new(core.Genesis)
 	genesisConfig.GasLimit = initialGas

cmd/evm/t8n_test.go

@@ -341,98 +341,6 @@ func lineIterator(path string) func() (string, error) {
 	}
 }
 
-// TestT8nTracing is a test that checks the tracing-output from t8n.
-func TestT8nTracing(t *testing.T) {
-	t.Parallel()
-	tt := new(testT8n)
-	tt.TestCmd = cmdtest.NewTestCmd(t, tt)
-	for i, tc := range []struct {
-		base           string
-		input          t8nInput
-		expExitCode    int
-		extraArgs      []string
-		expectedTraces []string
-	}{
-		{
-			base: "./testdata/31",
-			input: t8nInput{
-				"alloc.json", "txs.json", "env.json", "Cancun", "",
-			},
-			extraArgs:      []string{"--trace"},
-			expectedTraces: []string{"trace-0-0x88f5fbd1524731a81e49f637aa847543268a5aaf2a6b32a69d2c6d978c45dcfb.jsonl"},
-		},
-		{
-			base: "./testdata/31",
-			input: t8nInput{
-				"alloc.json", "txs.json", "env.json", "Cancun", "",
-			},
-			extraArgs: []string{"--trace.tracer", `
-{ 
-	result: function(){ 
-		return "hello world"
-	}, 
-	fault: function(){} 
-}`},
-			expectedTraces: []string{"trace-0-0x88f5fbd1524731a81e49f637aa847543268a5aaf2a6b32a69d2c6d978c45dcfb.json"},
-		},
-		{
-			base: "./testdata/32",
-			input: t8nInput{
-				"alloc.json", "txs.json", "env.json", "Paris", "",
-			},
-			extraArgs:      []string{"--trace", "--trace.callframes"},
-			expectedTraces: []string{"trace-0-0x47806361c0fa084be3caa18afe8c48156747c01dbdfc1ee11b5aecdbe4fcf23e.jsonl"},
-		},
-	} {
-		args := []string{"t8n"}
-		args = append(args, tc.input.get(tc.base)...)
-		// Place the output somewhere we can find it
-		outdir := t.TempDir()
-		args = append(args, "--output.basedir", outdir)
-		args = append(args, tc.extraArgs...)
-
-		var qArgs []string // quoted args for debugging purposes
-		for _, arg := range args {
-			if len(arg) == 0 {
-				qArgs = append(qArgs, `""`)
-			} else {
-				qArgs = append(qArgs, arg)
-			}
-		}
-		tt.Logf("args: %v\n", strings.Join(qArgs, " "))
-		tt.Run("evm-test", args...)
-		t.Log(string(tt.Output()))
-
-		// Compare the expected traces
-		for _, traceFile := range tc.expectedTraces {
-			haveFn := lineIterator(filepath.Join(outdir, traceFile))
-			wantFn := lineIterator(filepath.Join(tc.base, traceFile))
-
-			for line := 0; ; line++ {
-				want, wErr := wantFn()
-				have, hErr := haveFn()
-				if want != have {
-					t.Fatalf("test %d, trace %v, line %d\nwant: %v\nhave: %v\n",
-						i, traceFile, line, want, have)
-				}
-				if wErr != nil && hErr != nil {
-					break
-				}
-				if wErr != nil {
-					t.Fatal(wErr)
-				}
-				if hErr != nil {
-					t.Fatal(hErr)
-				}
-				t.Logf("%v\n", want)
-			}
-		}
-		if have, want := tt.ExitStatus(), tc.expExitCode; have != want {
-			t.Fatalf("test %d: wrong exit code, have %d, want %d", i, have, want)
-		}
-	}
-}
-
 type t9nInput struct {
 	inTxs  string
 	stFork string
@@ -672,6 +580,88 @@ func TestB11r(t *testing.T) {
 	}
 }
 
+func TestEvmRun(t *testing.T) {
+	t.Parallel()
+	tt := cmdtest.NewTestCmd(t, nil)
+	for i, tc := range []struct {
+		input      []string
+		wantStdout string
+		wantStderr string
+	}{
+		{ // json tracing
+			input:      []string{"run", "--trace", "--trace.format=json", "6040"},
+			wantStdout: "./testdata/evmrun/1.out.1.txt",
+			wantStderr: "./testdata/evmrun/1.out.2.txt",
+		},
+		{ // Same as above, using the deprecated --json
+			input:      []string{"run", "--json", "6040"},
+			wantStdout: "./testdata/evmrun/1.out.1.txt",
+			wantStderr: "./testdata/evmrun/1.out.2.txt",
+		},
+		{ // default tracing (struct)
+			input:      []string{"run", "--trace", "0x6040"},
+			wantStdout: "./testdata/evmrun/2.out.1.txt",
+			wantStderr: "./testdata/evmrun/2.out.2.txt",
+		},
+		{ // default tracing (struct), plus alloc-dump
+			input:      []string{"run", "--trace", "--dump", "0x6040"},
+			wantStdout: "./testdata/evmrun/3.out.1.txt",
+			//wantStderr: "./testdata/evmrun/3.out.2.txt",
+		},
+		{ // json-tracing, plus alloc-dump
+			input:      []string{"run", "--trace", "--trace.format=json", "--dump", "0x6040"},
+			wantStdout: "./testdata/evmrun/4.out.1.txt",
+			//wantStderr: "./testdata/evmrun/4.out.2.txt",
+		},
+		{ // md-tracing
+			input:      []string{"run", "--trace", "--trace.format=md", "0x6040"},
+			wantStdout: "./testdata/evmrun/5.out.1.txt",
+			wantStderr: "./testdata/evmrun/5.out.2.txt",
+		},
+		{ // statetest subcommand
+			input:      []string{"statetest", "./testdata/statetest.json"},
+			wantStdout: "./testdata/evmrun/6.out.1.txt",
+			wantStderr: "./testdata/evmrun/6.out.2.txt",
+		},
+		{ // statetest subcommand with output
+			input:      []string{"statetest", "--trace", "--trace.format=md", "./testdata/statetest.json"},
+			wantStdout: "./testdata/evmrun/7.out.1.txt",
+			wantStderr: "./testdata/evmrun/7.out.2.txt",
+		},
+		{ // statetest subcommand with output
+			input:      []string{"statetest", "--trace", "--trace.format=json", "./testdata/statetest.json"},
+			wantStdout: "./testdata/evmrun/8.out.1.txt",
+			wantStderr: "./testdata/evmrun/8.out.2.txt",
+		},
+	} {
+		tt.Logf("args: go run ./cmd/evm %v\n", strings.Join(tc.input, " "))
+		tt.Run("evm-test", tc.input...)
+
+		haveStdOut := tt.Output()
+		tt.WaitExit()
+		haveStdErr := tt.StderrText()
+
+		if have, wantFile := haveStdOut, tc.wantStdout; wantFile != "" {
+			want, err := os.ReadFile(wantFile)
+			if err != nil {
+				t.Fatalf("test %d: could not read expected output: %v", i, err)
+			}
+			if string(haveStdOut) != string(want) {
+				t.Fatalf("test %d, output wrong, have \n%v\nwant\n%v\n", i, string(have), string(want))
+			}
+		}
+		if have, wantFile := haveStdErr, tc.wantStderr; wantFile != "" {
+			want, err := os.ReadFile(wantFile)
+			if err != nil {
+				t.Fatalf("test %d: could not read expected output: %v", i, err)
+			}
+			if have != string(want) {
+				t.Fatalf("test %d, output wrong\nhave %q\nwant %q\n", i, have, string(want))
+			}
+		}
+	}
+}
+
 // cmpJson compares the JSON in two byte slices.
 func cmpJson(a, b []byte) (bool, error) {
 	var j, j2 interface{}
@@ -683,3 +673,93 @@ func cmpJson(a, b []byte) (bool, error) {
 	}
 	return reflect.DeepEqual(j2, j), nil
 }
+
+// TestEVMTracing is a test that checks the tracing-output from evm.
+func TestEVMTracing(t *testing.T) {
+	t.Parallel()
+	tt := cmdtest.NewTestCmd(t, nil)
+	for i, tc := range []struct {
+		base           string
+		input          []string
+		expectedTraces []string
+	}{
+		{
+			base: "./testdata/31",
+			input: []string{"t8n",
+				"--input.alloc=./testdata/31/alloc.json", "--input.txs=./testdata/31/txs.json",
+				"--input.env=./testdata/31/env.json", "--state.fork=Cancun",
+				"--trace",
+			},
+			expectedTraces: []string{"trace-0-0x88f5fbd1524731a81e49f637aa847543268a5aaf2a6b32a69d2c6d978c45dcfb.jsonl"},
+		},
+		{
+			base: "./testdata/31",
+			input: []string{"t8n",
+				"--input.alloc=./testdata/31/alloc.json", "--input.txs=./testdata/31/txs.json",
+				"--input.env=./testdata/31/env.json", "--state.fork=Cancun",
+				"--trace.tracer", `
+{ 
+	result: function(){ 
+		return "hello world"
+	}, 
+	fault: function(){} 
+}`,
+			},
+			expectedTraces: []string{"trace-0-0x88f5fbd1524731a81e49f637aa847543268a5aaf2a6b32a69d2c6d978c45dcfb.json"},
+		},
+		{
+			base: "./testdata/32",
+			input: []string{"t8n",
+				"--input.alloc=./testdata/32/alloc.json", "--input.txs=./testdata/32/txs.json",
+				"--input.env=./testdata/32/env.json", "--state.fork=Paris",
+				"--trace", "--trace.callframes",
+			},
+			expectedTraces: []string{"trace-0-0x47806361c0fa084be3caa18afe8c48156747c01dbdfc1ee11b5aecdbe4fcf23e.jsonl"},
+		},
+		// TODO, make it possible to run tracers on statetests, e.g:
+		//{
+		//			base: "./testdata/31",
+		//			input: []string{"statetest", "--trace", "--trace.tracer", `{
+		//	result: function(){
+		//		return "hello world"
+		//	},
+		//	fault: function(){}
+		//}`, "./testdata/statetest.json"},
+		//			expectedTraces: []string{"trace-0-0x88f5fbd1524731a81e49f637aa847543268a5aaf2a6b32a69d2c6d978c45dcfb.json"},
+		//		},
+	} {
+		// Place the output somewhere we can find it
+		outdir := t.TempDir()
+		args := append(tc.input, "--output.basedir", outdir)
+
+		tt.Run("evm-test", args...)
+		tt.Logf("args: go run ./cmd/evm %v\n", args)
+		tt.WaitExit()
+		//t.Log(string(tt.Output()))
+
+		// Compare the expected traces
+		for _, traceFile := range tc.expectedTraces {
+			haveFn := lineIterator(filepath.Join(outdir, traceFile))
+			wantFn := lineIterator(filepath.Join(tc.base, traceFile))
+
+			for line := 0; ; line++ {
+				want, wErr := wantFn()
+				have, hErr := haveFn()
+				if want != have {
+					t.Fatalf("test %d, trace %v, line %d\nwant: %v\nhave: %v\n",
+						i, traceFile, line, want, have)
+				}
+				if wErr != nil && hErr != nil {
+					break
+				}
+				if wErr != nil {
+					t.Fatal(wErr)
+				}
+				if hErr != nil {
+					t.Fatal(hErr)
+				}
+				//t.Logf("%v\n", want)
+			}
+		}
+	}
+}

cmd/evm/testdata/evmrun/1.out.1.txt

@@ -0,0 +1,3 @@
+{"pc":0,"op":96,"gas":"0x2540be400","gasCost":"0x3","memSize":0,"stack":[],"depth":1,"refund":0,"opName":"PUSH1"}
+{"pc":2,"op":0,"gas":"0x2540be3fd","gasCost":"0x0","memSize":0,"stack":["0x40"],"depth":1,"refund":0,"opName":"STOP"}
+{"output":"","gasUsed":"0x3"}

cmd/evm/testdata/evmrun/1.out.2.txt

@@ -0,0 +1,6 @@
+PUSH1           pc=00000000 gas=10000000000 cost=3
+
+STOP            pc=00000002 gas=9999999997 cost=0
+Stack:
+00000000  0x40
+

cmd/evm/testdata/evmrun/2.out.1.txt

@@ -0,0 +1,13 @@
+{
+    "root": "b444481d1367188172f8c6091e948aaa68bae763fd26d6b9e994306a66bf69f9",
+    "accounts": {
+        "pre(0x30d7a0694cb29af31b982480e11d7ebb003a3fca4026939149071f014689b142)": {
+            "balance": "0",
+            "nonce": 0,
+            "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+            "codeHash": "0x3e48ef54b89079a075f3b8fc253c657a86b110a7aed3568c1517b10edf2c3eb6",
+            "code": "0x6040",
+            "key": "0x30d7a0694cb29af31b982480e11d7ebb003a3fca4026939149071f014689b142"
+        }
+    }
+}

cmd/evm/testdata/evmrun/2.out.2.txt

@@ -0,0 +1,9 @@
+PUSH1           pc=00000000 gas=10000000000 cost=3
+
+STOP            pc=00000002 gas=9999999997 cost=0
+Stack:
+00000000  0x40
+
+INFO [12-03|10:37:15.827] Trie dumping started                     root=b44448..bf69f9
+WARN [12-03|10:37:15.827] Dump incomplete due to missing preimages missing=1
+INFO [12-03|10:37:15.827] Trie dumping complete                    accounts=1 elapsed="163.513µs"