Skip to content

Commit 0e69530

Browse files
authored
all: improve ETA calculation across all progress indicators (#32521)
### Summary Fixes long-standing ETA calculation errors in progress indicators that have been present since February 2021. The current implementation produces increasingly inaccurate estimates due to integer division precision loss. ### Problem https://github.com/ethereum/go-ethereum/blob/3aeccadd04aee2d18bdb77826f86b1ca000d3b67/triedb/pathdb/history_indexer.go#L541-L553 The ETA calculation has two critical issues: 1. **Integer division precision loss**: `speed` is calculated as `uint64` 2. **Off-by-one**: `speed` uses `+ 1`(2 times) to avoid division by zero, however it makes mistake in the final calculation This results in wildly inaccurate time estimates that don't improve as progress continues. ### Example Current output during state history indexing: ``` lvl=info msg="Indexing state history" processed=16858580 left=41802252 elapsed=18h22m59.848s eta=11h36m42.252s ``` **Expected calculation:** - Speed: 16858580 ÷ 66179848ms = 0.255 blocks/ms - ETA: 41802252 ÷ 0.255 = ~45.6 hours **Current buggy calculation:** - Speed: rounds to 1 block/ms - ETA: 41802252 ÷ 1 = ~11.6 hours ❌ ### Solution - Created centralized `CalculateETA()` function in common package - Replaced all 8 duplicate code copies across the codebase ### Testing Verified accurate ETA calculations during archive node reindexing with significantly improved time estimates.
1 parent 0cde527 commit 0e69530

File tree

6 files changed

+104
-26
lines changed

6 files changed

+104
-26
lines changed

common/eta.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2025 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package common
18+
19+
import "time"
20+
21+
// CalculateETA calculates the estimated remaining time based on the
22+
// number of finished task, remaining task, and the time cost for finished task.
23+
func CalculateETA(done, left uint64, elapsed time.Duration) time.Duration {
24+
if done == 0 || elapsed.Milliseconds() == 0 {
25+
return 0
26+
}
27+
28+
speed := float64(done) / float64(elapsed.Milliseconds())
29+
return time.Duration(float64(left)/speed) * time.Millisecond
30+
}

common/eta_test.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2025 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package common
18+
19+
import (
20+
"testing"
21+
"time"
22+
)
23+
24+
func TestCalculateETA(t *testing.T) {
25+
type args struct {
26+
done uint64
27+
left uint64
28+
elapsed time.Duration
29+
}
30+
tests := []struct {
31+
name string
32+
args args
33+
want time.Duration
34+
}{
35+
{
36+
name: "zero done",
37+
args: args{done: 0, left: 100, elapsed: time.Second},
38+
want: 0,
39+
},
40+
{
41+
name: "zero elapsed",
42+
args: args{done: 1, left: 100, elapsed: 0},
43+
want: 0,
44+
},
45+
{
46+
name: "@Jolly23 's case",
47+
args: args{done: 16858580, left: 41802252, elapsed: 66179848 * time.Millisecond},
48+
want: 164098440 * time.Millisecond,
49+
// wrong msg: msg="Indexing state history" processed=16858580 left=41802252 elapsed=18h22m59.848s eta=11h36m42.252s
50+
// should be around 45.58 hours
51+
},
52+
}
53+
for _, tt := range tests {
54+
t.Run(tt.name, func(t *testing.T) {
55+
if got := CalculateETA(tt.args.done, tt.args.left, tt.args.elapsed); got != tt.want {
56+
t.Errorf("CalculateETA() = %v ms, want %v ms", got.Milliseconds(), tt.want)
57+
}
58+
})
59+
}
60+
}

core/state/pruner/pruner.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,8 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
160160

161161
var eta time.Duration // Realistically will never remain uninited
162162
if done := binary.BigEndian.Uint64(key[:8]); done > 0 {
163-
var (
164-
left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
165-
speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero
166-
)
167-
eta = time.Duration(left/speed) * time.Millisecond
163+
left := math.MaxUint64 - binary.BigEndian.Uint64(key[:8])
164+
eta = common.CalculateETA(done, left, time.Since(pstart))
168165
}
169166
if time.Since(logged) > 8*time.Second {
170167
log.Info("Pruning state data", "nodes", count, "skipped", skipped, "size", size,

core/state/snapshot/conversion.go

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -171,20 +171,16 @@ func (stat *generateStats) report() {
171171
// If there's progress on the account trie, estimate the time to finish crawling it
172172
if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
173173
var (
174-
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
175-
speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
176-
eta = time.Duration(left/speed) * time.Millisecond
174+
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
175+
eta = common.CalculateETA(done, left, time.Since(stat.start))
177176
)
178177
// If there are large contract crawls in progress, estimate their finish time
179178
for acc, head := range stat.slotsHead {
180179
start := stat.slotsStart[acc]
181180
if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
182-
var (
183-
left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
184-
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
185-
)
181+
left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
186182
// Override the ETA if larger than the largest until now
187-
if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
183+
if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA {
188184
eta = slotETA
189185
}
190186
}

triedb/pathdb/history_indexer.go

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -543,12 +543,10 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
543543
logged = time.Now()
544544

545545
var (
546-
left = lastID - current + 1
547-
done = current - beginID
548-
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
546+
left = lastID - current + 1
547+
done = current - beginID
549548
)
550-
// Override the ETA if larger than the largest until now
551-
eta := time.Duration(left/speed) * time.Millisecond
549+
eta := common.CalculateETA(done, left, time.Since(start))
552550
log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
553551
}
554552
}

triedb/pathdb/verifier.go

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -166,20 +166,17 @@ func (stat *generateStats) report() {
166166
// If there's progress on the account trie, estimate the time to finish crawling it
167167
if done := binary.BigEndian.Uint64(stat.head[:8]) / stat.accounts; done > 0 {
168168
var (
169-
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
170-
speed = done/uint64(time.Since(stat.start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
171-
eta = time.Duration(left/speed) * time.Millisecond
169+
left = (math.MaxUint64 - binary.BigEndian.Uint64(stat.head[:8])) / stat.accounts
170+
eta = common.CalculateETA(done, left, time.Since(stat.start))
172171
)
173172
// If there are large contract crawls in progress, estimate their finish time
174173
for acc, head := range stat.slotsHead {
175174
start := stat.slotsStart[acc]
176175
if done := binary.BigEndian.Uint64(head[:8]); done > 0 {
177-
var (
178-
left = math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
179-
speed = done/uint64(time.Since(start)/time.Millisecond+1) + 1 // +1s to avoid division by zero
180-
)
176+
left := math.MaxUint64 - binary.BigEndian.Uint64(head[:8])
177+
181178
// Override the ETA if larger than the largest until now
182-
if slotETA := time.Duration(left/speed) * time.Millisecond; eta < slotETA {
179+
if slotETA := common.CalculateETA(done, left, time.Since(start)); eta < slotETA {
183180
eta = slotETA
184181
}
185182
}

0 commit comments

Comments
 (0)