Skip to content

Commit 5dd2ce0

Browse files
holimanrjl493456442
authored and
jakub-freebit
committed
cmd/evm: refactor handling output-files for t8n (ethereum#30854)
As part of trying to make the inputs and outputs of the evm subcommands more streamlined and aligned, this PR modifies how `evm t8n` manages output-files. Previously, we do a kind of wonky thing where between each transaction, we invoke a `getTracer` closure. In that closure, we create a new output-file, a tracer, and then make the tracer stream output to the file. We also fiddle a bit to ensure that the file becomes properly closed. It is a kind of hacky solution we have in place. This PR changes it, so that from the execution-pipeline point of view, we have just a regular tracer. No fiddling with re-setting it or closing files. That particular tracer, however, is a bit special: it takes care of creating new files per transaction (in the tx-start-hook) and closing (on tx-end-hook). Also instantiating the right type of underlying tracer, which can be a json-logger or a custom tracer. --------- Co-authored-by: Gary Rong <[email protected]>
1 parent 286515f commit 5dd2ce0

14 files changed

+260
-118
lines changed

‎cmd/evm/internal/t8ntool/execution.go‎

Lines changed: 7 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,7 @@
1717
package t8ntool
1818

1919
import (
20-
"encoding/json"
2120
"fmt"
22-
"io"
2321
"math/big"
2422

2523
"github.com/ethereum/go-ethereum/common"
@@ -35,7 +33,6 @@ import (
3533
"github.com/ethereum/go-ethereum/core/types"
3634
"github.com/ethereum/go-ethereum/core/vm"
3735
"github.com/ethereum/go-ethereum/crypto"
38-
"github.com/ethereum/go-ethereum/eth/tracers"
3936
"github.com/ethereum/go-ethereum/ethdb"
4037
"github.com/ethereum/go-ethereum/log"
4138
"github.com/ethereum/go-ethereum/params"
@@ -130,9 +127,7 @@ type rejectedTx struct {
130127
}
131128

132129
// Apply applies a set of transactions to a pre-state
133-
func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
134-
txIt txIterator, miningReward int64,
135-
getTracerFn func(txIndex int, txHash common.Hash, chainConfig *params.ChainConfig) (*tracers.Tracer, io.WriteCloser, error)) (*state.StateDB, *ExecutionResult, []byte, error) {
130+
func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txIt txIterator, miningReward int64) (*state.StateDB, *ExecutionResult, []byte, error) {
136131
// Capture errors for BLOCKHASH operation, if we haven't been supplied the
137132
// required blockhashes
138133
var hashError error
@@ -241,23 +236,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
241236
continue
242237
}
243238
}
244-
tracer, traceOutput, err := getTracerFn(txIndex, tx.Hash(), chainConfig)
245-
if err != nil {
246-
return nil, nil, nil, err
247-
}
248-
// TODO (rjl493456442) it's a bit weird to reset the tracer in the
249-
// middle of block execution, please improve it somehow.
250-
if tracer != nil {
251-
evm.SetTracer(tracer.Hooks)
252-
}
253239
statedb.SetTxContext(tx.Hash(), txIndex)
254-
255240
var (
256241
snapshot = statedb.Snapshot()
257242
prevGas = gaspool.Gas()
258243
)
259-
if tracer != nil && tracer.OnTxStart != nil {
260-
tracer.OnTxStart(evm.GetVMContext(), tx, msg.From)
244+
if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxStart != nil {
245+
evm.Config.Tracer.OnTxStart(evm.GetVMContext(), tx, msg.From)
261246
}
262247
// (ret []byte, usedGas uint64, failed bool, err error)
263248
msgResult, err := core.ApplyMessage(evm, msg, gaspool)
@@ -266,13 +251,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
266251
log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err)
267252
rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})
268253
gaspool.SetGas(prevGas)
269-
if tracer != nil {
270-
if tracer.OnTxEnd != nil {
271-
tracer.OnTxEnd(nil, err)
272-
}
273-
if err := writeTraceResult(tracer, traceOutput); err != nil {
274-
log.Warn("Error writing tracer output", "err", err)
275-
}
254+
if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxEnd != nil {
255+
evm.Config.Tracer.OnTxEnd(nil, err)
276256
}
277257
continue
278258
}
@@ -316,13 +296,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
316296
//receipt.BlockNumber
317297
receipt.TransactionIndex = uint(txIndex)
318298
receipts = append(receipts, receipt)
319-
if tracer != nil {
320-
if tracer.Hooks.OnTxEnd != nil {
321-
tracer.Hooks.OnTxEnd(receipt, nil)
322-
}
323-
if err = writeTraceResult(tracer, traceOutput); err != nil {
324-
log.Warn("Error writing tracer output", "err", err)
325-
}
299+
if evm.Config.Tracer != nil && evm.Config.Tracer.OnTxEnd != nil {
300+
evm.Config.Tracer.OnTxEnd(receipt, nil)
326301
}
327302
}
328303

@@ -468,16 +443,3 @@ func calcDifficulty(config *params.ChainConfig, number, currentTime, parentTime
468443
}
469444
return ethash.CalcDifficulty(config, currentTime, parent)
470445
}
471-
472-
func writeTraceResult(tracer *tracers.Tracer, f io.WriteCloser) error {
473-
defer f.Close()
474-
result, err := tracer.GetResult()
475-
if err != nil || result == nil {
476-
return err
477-
}
478-
err = json.NewEncoder(f).Encode(result)
479-
if err != nil {
480-
return err
481-
}
482-
return nil
483-
}

‎cmd/evm/internal/t8ntool/file_tracer.go‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
// Copyright 2024 The go-ethereum Authors
2+
// This file is part of go-ethereum.
3+
//
4+
// go-ethereum is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// go-ethereum is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU General Public License
15+
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package t8ntool
18+
19+
import (
20+
"encoding/json"
21+
"fmt"
22+
"io"
23+
"math/big"
24+
"os"
25+
"path/filepath"
26+
27+
"github.com/ethereum/go-ethereum/common"
28+
"github.com/ethereum/go-ethereum/core/tracing"
29+
"github.com/ethereum/go-ethereum/core/types"
30+
"github.com/ethereum/go-ethereum/eth/tracers"
31+
"github.com/ethereum/go-ethereum/log"
32+
)
33+
34+
// fileWritingTracer wraps either a tracer or a logger. On tx start,
35+
// it instantiates a tracer/logger, creates a new file to direct output to,
36+
// and on tx end it closes the file.
37+
type fileWritingTracer struct {
38+
txIndex int // transaction counter
39+
inner *tracing.Hooks // inner hooks
40+
destination io.WriteCloser // the currently open file (if any)
41+
baseDir string // baseDir to write output-files to
42+
suffix string // suffix is the suffix to use when creating files
43+
44+
// for custom tracing
45+
getResult func() (json.RawMessage, error)
46+
}
47+
48+
func (l *fileWritingTracer) Write(p []byte) (n int, err error) {
49+
if l.destination != nil {
50+
return l.destination.Write(p)
51+
}
52+
log.Warn("Tracer wrote to non-existing output")
53+
// It is tempting to return an error here, however, the json encoder
54+
// will no retry writing to an io.Writer once it has returned an error once.
55+
// Therefore, we must squash the error.
56+
return n, nil
57+
}
58+
59+
// newFileWriter creates a set of hooks which wraps inner hooks (typically a logger),
60+
// and writes the output to a file, one file per transaction.
61+
func newFileWriter(baseDir string, innerFn func(out io.Writer) *tracing.Hooks) *tracing.Hooks {
62+
t := &fileWritingTracer{
63+
baseDir: baseDir,
64+
suffix: "jsonl",
65+
}
66+
t.inner = innerFn(t) // instantiate the inner tracer
67+
return t.hooks()
68+
}
69+
70+
// newResultWriter creates a set of hooks wraps and invokes an underlying tracer,
71+
// and writes the result (getResult-output) to file, one per transaction.
72+
func newResultWriter(baseDir string, tracer *tracers.Tracer) *tracing.Hooks {
73+
t := &fileWritingTracer{
74+
baseDir: baseDir,
75+
getResult: tracer.GetResult,
76+
inner: tracer.Hooks,
77+
suffix: "json",
78+
}
79+
return t.hooks()
80+
}
81+
82+
// OnTxStart creates a new output-file specific for this transaction, and invokes
83+
// the inner OnTxStart handler.
84+
func (l *fileWritingTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from common.Address) {
85+
// Open a new file, or print a warning log if it's failed
86+
fname := filepath.Join(l.baseDir, fmt.Sprintf("trace-%d-%v.%v", l.txIndex, tx.Hash().String(), l.suffix))
87+
traceFile, err := os.Create(fname)
88+
if err != nil {
89+
log.Warn("Failed creating trace-file", "err", err)
90+
} else {
91+
log.Info("Created tracing-file", "path", fname)
92+
l.destination = traceFile
93+
}
94+
if l.inner != nil && l.inner.OnTxStart != nil {
95+
l.inner.OnTxStart(env, tx, from)
96+
}
97+
}
98+
99+
// OnTxEnd writes result (if getResult exist), closes any currently open output-file,
100+
// and invokes the inner OnTxEnd handler.
101+
func (l *fileWritingTracer) OnTxEnd(receipt *types.Receipt, err error) {
102+
if l.inner != nil && l.inner.OnTxEnd != nil {
103+
l.inner.OnTxEnd(receipt, err)
104+
}
105+
if l.getResult != nil && l.destination != nil {
106+
if result, err := l.getResult(); result != nil {
107+
json.NewEncoder(l.destination).Encode(result)
108+
} else {
109+
log.Warn("Error obtaining tracer result", "err", err)
110+
}
111+
l.destination.Close()
112+
l.destination = nil
113+
}
114+
l.txIndex++
115+
}
116+
117+
func (l *fileWritingTracer) hooks() *tracing.Hooks {
118+
return &tracing.Hooks{
119+
OnTxStart: l.OnTxStart,
120+
OnTxEnd: l.OnTxEnd,
121+
OnEnter: func(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) {
122+
if l.inner != nil && l.inner.OnEnter != nil {
123+
l.inner.OnEnter(depth, typ, from, to, input, gas, value)
124+
}
125+
},
126+
OnExit: func(depth int, output []byte, gasUsed uint64, err error, reverted bool) {
127+
if l.inner != nil && l.inner.OnExit != nil {
128+
l.inner.OnExit(depth, output, gasUsed, err, reverted)
129+
}
130+
},
131+
OnOpcode: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) {
132+
if l.inner != nil && l.inner.OnOpcode != nil {
133+
l.inner.OnOpcode(pc, op, gas, cost, scope, rData, depth, err)
134+
}
135+
},
136+
OnFault: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, depth int, err error) {
137+
if l.inner != nil && l.inner.OnFault != nil {
138+
l.inner.OnFault(pc, op, gas, cost, scope, depth, err)
139+
}
140+
},
141+
OnSystemCallStart: func() {
142+
if l.inner != nil && l.inner.OnSystemCallStart != nil {
143+
l.inner.OnSystemCallStart()
144+
}
145+
},
146+
OnSystemCallEnd: func() {
147+
if l.inner != nil && l.inner.OnSystemCallEnd != nil {
148+
l.inner.OnSystemCallEnd()
149+
}
150+
},
151+
}
152+
}

‎cmd/evm/internal/t8ntool/transaction.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,10 @@ func (r *result) MarshalJSON() ([]byte, error) {
6464
}
6565

6666
func Transaction(ctx *cli.Context) error {
67-
var (
68-
err error
69-
)
7067
// We need to load the transactions. May be either in stdin input or in files.
7168
// Check if anything needs to be read from stdin
7269
var (
70+
err error
7371
txStr = ctx.String(InputTxsFlag.Name)
7472
inputData = &input{}
7573
chainConfig *params.ChainConfig
@@ -82,6 +80,7 @@ func Transaction(ctx *cli.Context) error {
8280
}
8381
// Set the chain id
8482
chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name))
83+
8584
var body hexutil.Bytes
8685
if txStr == stdinSelector {
8786
decoder := json.NewDecoder(os.Stdin)
@@ -107,6 +106,7 @@ func Transaction(ctx *cli.Context) error {
107106
}
108107
}
109108
signer := types.MakeSigner(chainConfig, new(big.Int), 0)
109+
110110
// We now have the transactions in 'body', which is supposed to be an
111111
// rlp list of transactions
112112
it, err := rlp.NewListIterator([]byte(body))

‎cmd/evm/internal/t8ntool/transition.go‎

Lines changed: 28 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -82,58 +82,10 @@ type input struct {
8282
}
8383

8484
func Transition(ctx *cli.Context) error {
85-
var getTracer = func(txIndex int, txHash common.Hash, chainConfig *params.ChainConfig) (*tracers.Tracer, io.WriteCloser, error) {
86-
return nil, nil, nil
87-
}
88-
8985
baseDir, err := createBasedir(ctx)
9086
if err != nil {
9187
return NewError(ErrorIO, fmt.Errorf("failed creating output basedir: %v", err))
9288
}
93-
94-
if ctx.Bool(TraceFlag.Name) { // JSON opcode tracing
95-
// Configure the EVM logger
96-
logConfig := &logger.Config{
97-
DisableStack: ctx.Bool(TraceDisableStackFlag.Name),
98-
EnableMemory: ctx.Bool(TraceEnableMemoryFlag.Name),
99-
EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name),
100-
}
101-
getTracer = func(txIndex int, txHash common.Hash, _ *params.ChainConfig) (*tracers.Tracer, io.WriteCloser, error) {
102-
traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.jsonl", txIndex, txHash.String())))
103-
if err != nil {
104-
return nil, nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err))
105-
}
106-
var l *tracing.Hooks
107-
if ctx.Bool(TraceEnableCallFramesFlag.Name) {
108-
l = logger.NewJSONLoggerWithCallFrames(logConfig, traceFile)
109-
} else {
110-
l = logger.NewJSONLogger(logConfig, traceFile)
111-
}
112-
tracer := &tracers.Tracer{
113-
Hooks: l,
114-
// jsonLogger streams out result to file.
115-
GetResult: func() (json.RawMessage, error) { return nil, nil },
116-
Stop: func(err error) {},
117-
}
118-
return tracer, traceFile, nil
119-
}
120-
} else if ctx.IsSet(TraceTracerFlag.Name) {
121-
var config json.RawMessage
122-
if ctx.IsSet(TraceTracerConfigFlag.Name) {
123-
config = []byte(ctx.String(TraceTracerConfigFlag.Name))
124-
}
125-
getTracer = func(txIndex int, txHash common.Hash, chainConfig *params.ChainConfig) (*tracers.Tracer, io.WriteCloser, error) {
126-
traceFile, err := os.Create(filepath.Join(baseDir, fmt.Sprintf("trace-%d-%v.json", txIndex, txHash.String())))
127-
if err != nil {
128-
return nil, nil, NewError(ErrorIO, fmt.Errorf("failed creating trace-file: %v", err))
129-
}
130-
tracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), nil, config, chainConfig)
131-
if err != nil {
132-
return nil, nil, NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %w", err))
133-
}
134-
return tracer, traceFile, nil
135-
}
136-
}
13789
// We need to load three things: alloc, env and transactions. May be either in
13890
// stdin input or in files.
13991
// Check if anything needs to be read from stdin
@@ -179,6 +131,7 @@ func Transition(ctx *cli.Context) error {
179131
chainConfig = cConf
180132
vmConfig.ExtraEips = extraEips
181133
}
134+
182135
// Set the chain id
183136
chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name))
184137

@@ -197,8 +150,34 @@ func Transition(ctx *cli.Context) error {
197150
if err := applyCancunChecks(&prestate.Env, chainConfig); err != nil {
198151
return err
199152
}
153+
154+
// Configure tracer
155+
if ctx.IsSet(TraceTracerFlag.Name) { // Custom tracing
156+
config := json.RawMessage(ctx.String(TraceTracerConfigFlag.Name))
157+
tracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name),
158+
nil, config, chainConfig)
159+
if err != nil {
160+
return NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %v", err))
161+
}
162+
vmConfig.Tracer = newResultWriter(baseDir, tracer)
163+
} else if ctx.Bool(TraceFlag.Name) { // JSON opcode tracing
164+
logConfig := &logger.Config{
165+
DisableStack: ctx.Bool(TraceDisableStackFlag.Name),
166+
EnableMemory: ctx.Bool(TraceEnableMemoryFlag.Name),
167+
EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name),
168+
}
169+
if ctx.Bool(TraceEnableCallFramesFlag.Name) {
170+
vmConfig.Tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks {
171+
return logger.NewJSONLoggerWithCallFrames(logConfig, out)
172+
})
173+
} else {
174+
vmConfig.Tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks {
175+
return logger.NewJSONLogger(logConfig, out)
176+
})
177+
}
178+
}
200179
// Run the test and aggregate the result
201-
s, result, body, err := prestate.Apply(vmConfig, chainConfig, txIt, ctx.Int64(RewardFlag.Name), getTracer)
180+
s, result, body, err := prestate.Apply(vmConfig, chainConfig, txIt, ctx.Int64(RewardFlag.Name))
202181
if err != nil {
203182
return err
204183
}

0 commit comments

Comments
 (0)