Skip to content

Commit 2d3704c

Browse files
authored
core/stateless: add vmwitnessstats cli flag to report leaf stats + log to console (#32619)
The format that is currently reported by the chain isn't very useful, as it gives an average for ALL the nodes, and not only the leaves, which skews the results. Also, until now there was no way to activate the reporting of errors. We also decided that metrics weren't the right tool to report this data, so we decided to dump it to the console if the flag is enabled. A better system should be built, but for now, printing to the logs does the job.
1 parent f6ba50b commit 2d3704c

File tree

8 files changed

+118
-2
lines changed

8 files changed

+118
-2
lines changed

cmd/geth/main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ var (
133133
utils.VMEnableDebugFlag,
134134
utils.VMTraceFlag,
135135
utils.VMTraceJsonConfigFlag,
136+
utils.VMWitnessStatsFlag,
137+
utils.VMStatelessSelfValidationFlag,
136138
utils.NetworkIdFlag,
137139
utils.EthStatsURLFlag,
138140
utils.GpoBlocksFlag,

cmd/utils/flags.go

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,16 @@ var (
571571
Value: "{}",
572572
Category: flags.VMCategory,
573573
}
574+
VMWitnessStatsFlag = &cli.BoolFlag{
575+
Name: "vmwitnessstats",
576+
Usage: "Enable collection of witness trie access statistics (automatically enables witness generation)",
577+
Category: flags.VMCategory,
578+
}
579+
VMStatelessSelfValidationFlag = &cli.BoolFlag{
580+
Name: "stateless-self-validation",
581+
Usage: "Generate execution witnesses and self-check against them (testing purpose)",
582+
Category: flags.VMCategory,
583+
}
574584
// API options.
575585
RPCGlobalGasCapFlag = &cli.Uint64Flag{
576586
Name: "rpc.gascap",
@@ -1707,6 +1717,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
17071717
if ctx.IsSet(VMEnableDebugFlag.Name) {
17081718
cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name)
17091719
}
1720+
if ctx.IsSet(VMWitnessStatsFlag.Name) {
1721+
cfg.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name)
1722+
}
1723+
if ctx.IsSet(VMStatelessSelfValidationFlag.Name) {
1724+
cfg.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name)
1725+
}
1726+
// Auto-enable StatelessSelfValidation when witness stats are enabled
1727+
if ctx.Bool(VMWitnessStatsFlag.Name) {
1728+
cfg.StatelessSelfValidation = true
1729+
}
17101730

17111731
if ctx.IsSet(RPCGlobalGasCapFlag.Name) {
17121732
cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name)
@@ -2243,6 +2263,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
22432263
}
22442264
vmcfg := vm.Config{
22452265
EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name),
2266+
EnableWitnessStats: ctx.Bool(VMWitnessStatsFlag.Name),
2267+
StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name),
22462268
}
22472269
if ctx.IsSet(VMTraceFlag.Name) {
22482270
if name := ctx.String(VMTraceFlag.Name); name != "" {

core/blockchain.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2157,7 +2157,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s
21572157
}
21582158
// Report the collected witness statistics
21592159
if witnessStats != nil {
2160-
witnessStats.ReportMetrics()
2160+
witnessStats.ReportMetrics(block.NumberU64())
21612161
}
21622162

21632163
// Update the metrics touched during block commit

core/stateless/stats.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@
1717
package stateless
1818

1919
import (
20+
"encoding/json"
2021
"maps"
2122
"slices"
2223
"sort"
2324
"strconv"
2425
"strings"
2526

2627
"github.com/ethereum/go-ethereum/common"
28+
"github.com/ethereum/go-ethereum/log"
2729
"github.com/ethereum/go-ethereum/metrics"
2830
)
2931

@@ -70,7 +72,19 @@ func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) {
7072
}
7173

7274
// ReportMetrics reports the collected statistics to the global metrics registry.
73-
func (s *WitnessStats) ReportMetrics() {
75+
func (s *WitnessStats) ReportMetrics(blockNumber uint64) {
76+
// Encode the metrics as JSON for easier consumption
77+
accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves)
78+
storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves)
79+
80+
// Log account trie depth statistics
81+
log.Info("Account trie depth stats",
82+
"block", blockNumber,
83+
"leavesAtDepth", string(accountLeavesJson))
84+
log.Info("Storage trie depth stats",
85+
"block", blockNumber,
86+
"leavesAtDepth", string(storageLeavesJson))
87+
7488
for i := 0; i < 16; i++ {
7589
accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i])
7690
storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i])

core/stateless/stats_test.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,64 @@ func TestWitnessStatsAdd(t *testing.T) {
140140
}
141141
}
142142

143+
func TestWitnessStatsMinMax(t *testing.T) {
144+
stats := NewWitnessStats()
145+
146+
// Add some account trie nodes with varying depths
147+
stats.Add(map[string][]byte{
148+
"a": []byte("data1"),
149+
"ab": []byte("data2"),
150+
"abc": []byte("data3"),
151+
"abcd": []byte("data4"),
152+
"abcde": []byte("data5"),
153+
}, common.Hash{})
154+
155+
// Only "abcde" is a leaf (depth 5)
156+
for i, v := range stats.accountTrieLeaves {
157+
if v != 0 && i != 5 {
158+
t.Errorf("leaf found at invalid depth %d", i)
159+
}
160+
}
161+
162+
// Add more leaves with different depths
163+
stats.Add(map[string][]byte{
164+
"x": []byte("data6"),
165+
"yz": []byte("data7"),
166+
}, common.Hash{})
167+
168+
// Now we have leaves at depths 1, 2, and 5
169+
for i, v := range stats.accountTrieLeaves {
170+
if v != 0 && (i != 5 && i != 2 && i != 1) {
171+
t.Errorf("leaf found at invalid depth %d", i)
172+
}
173+
}
174+
}
175+
176+
func TestWitnessStatsAverage(t *testing.T) {
177+
stats := NewWitnessStats()
178+
179+
// Add nodes that will create leaves at depths 2, 3, and 4
180+
stats.Add(map[string][]byte{
181+
"aa": []byte("data1"),
182+
"bb": []byte("data2"),
183+
"ccc": []byte("data3"),
184+
"dddd": []byte("data4"),
185+
}, common.Hash{})
186+
187+
// All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples
188+
expectedAvg := int64(11) / int64(4)
189+
var actualAvg, totalSamples int64
190+
for i, c := range stats.accountTrieLeaves {
191+
actualAvg += c * int64(i)
192+
totalSamples += c
193+
}
194+
actualAvg = actualAvg / totalSamples
195+
196+
if actualAvg != expectedAvg {
197+
t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg)
198+
}
199+
}
200+
143201
func BenchmarkWitnessStatsAdd(b *testing.B) {
144202
// Create a realistic trie node structure
145203
nodes := make(map[string][]byte)

eth/backend.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
235235
TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)),
236236
VmConfig: vm.Config{
237237
EnablePreimageRecording: config.EnablePreimageRecording,
238+
EnableWitnessStats: config.EnableWitnessStats,
239+
StatelessSelfValidation: config.StatelessSelfValidation,
238240
},
239241
// Enables file journaling for the trie database. The journal files will be stored
240242
// within the data directory. The corresponding paths will be either:

eth/ethconfig/config.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,12 @@ type Config struct {
144144
// Enables tracking of SHA3 preimages in the VM
145145
EnablePreimageRecording bool
146146

147+
// Enables collection of witness trie access statistics
148+
EnableWitnessStats bool
149+
150+
// Generate execution witnesses and self-check against them (testing purpose)
151+
StatelessSelfValidation bool
152+
147153
// Enables tracking of state size
148154
EnableStateSizeTracking bool
149155

eth/ethconfig/gen_config.go

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)