Skip to content

Commit dde2718

Browse files
zhangchiqingzhangchiqing
authored
Merge pull request #6830 from onflow/leo/verify-execution-result-concurrent
[Util] add nWorker to verify execution result
2 parents 45950a7 + 05c552f commit dde2718

File tree

3 files changed

+190
-14
lines changed

3 files changed

+190
-14
lines changed

‎cmd/util/cmd/verify_execution_result/cmd.go‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ var (
1818
flagChunkDataPackDir string
1919
flagChain string
2020
flagFromTo string
21+
flagWorkerCount uint // number of workers to verify the blocks concurrently
2122
)
2223

2324
// # verify the last 100 sealed blocks
@@ -47,12 +48,20 @@ func init() {
4748

4849
Cmd.Flags().StringVar(&flagFromTo, "from_to", "",
4950
"the height range to verify blocks (inclusive), i.e, 1-1000, 1000-2000, 2000-3000, etc.")
51+
52+
Cmd.Flags().UintVar(&flagWorkerCount, "worker_count", 1,
53+
"number of workers to use for verification, default is 1")
54+
5055
}
5156

5257
func run(*cobra.Command, []string) {
5358
chainID := flow.ChainID(flagChain)
5459
_ = chainID.Chain()
5560

61+
if flagWorkerCount < 1 {
62+
log.Fatal().Msgf("worker count must be at least 1, but got %v", flagWorkerCount)
63+
}
64+
5665
lg := log.With().
5766
Str("chain", string(chainID)).
5867
Str("datadir", flagDatadir).
@@ -66,15 +75,15 @@ func run(*cobra.Command, []string) {
6675
}
6776

6877
lg.Info().Msgf("verifying range from %d to %d", from, to)
69-
err = verifier.VerifyRange(from, to, chainID, flagDatadir, flagChunkDataPackDir)
78+
err = verifier.VerifyRange(from, to, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount)
7079
if err != nil {
7180
lg.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to)
7281
}
7382
lg.Info().Msgf("successfully verified range from %d to %d", from, to)
7483

7584
} else {
7685
lg.Info().Msgf("verifying last %d sealed blocks", flagLastK)
77-
err := verifier.VerifyLastKHeight(flagLastK, chainID, flagDatadir, flagChunkDataPackDir)
86+
err := verifier.VerifyLastKHeight(flagLastK, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount)
7887
if err != nil {
7988
lg.Fatal().Err(err).Msg("could not verify last k height")
8089
}

‎engine/verification/verifier/verifiers.go‎

Lines changed: 94 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package verifier
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
7+
"sync"
68

79
"github.com/rs/zerolog"
810
"github.com/rs/zerolog/log"
@@ -25,7 +27,7 @@ import (
2527
// It assumes the latest sealed block has been executed, and the chunk data packs have not been
2628
// pruned.
2729
// Note, it returns nil if certain block is not executed, in this case warning will be logged
28-
func VerifyLastKHeight(k uint64, chainID flow.ChainID, protocolDataDir string, chunkDataPackDir string) (err error) {
30+
func VerifyLastKHeight(k uint64, chainID flow.ChainID, protocolDataDir string, chunkDataPackDir string, nWorker uint) (err error) {
2931
closer, storages, chunkDataPacks, state, verifier, err := initStorages(chainID, protocolDataDir, chunkDataPackDir)
3032
if err != nil {
3133
return fmt.Errorf("could not init storages: %w", err)
@@ -62,12 +64,9 @@ func VerifyLastKHeight(k uint64, chainID flow.ChainID, protocolDataDir string, c
6264

6365
log.Info().Msgf("verifying blocks from %d to %d", from, to)
6466

65-
for height := from; height <= to; height++ {
66-
log.Info().Uint64("height", height).Msg("verifying height")
67-
err := verifyHeight(height, storages.Headers, chunkDataPacks, storages.Results, state, verifier)
68-
if err != nil {
69-
return fmt.Errorf("could not verify height %d: %w", height, err)
70-
}
67+
err = verifyConcurrently(from, to, nWorker, storages.Headers, chunkDataPacks, storages.Results, state, verifier, verifyHeight)
68+
if err != nil {
69+
return err
7170
}
7271

7372
return nil
@@ -79,6 +78,7 @@ func VerifyRange(
7978
from, to uint64,
8079
chainID flow.ChainID,
8180
protocolDataDir string, chunkDataPackDir string,
81+
nWorker uint,
8282
) (err error) {
8383
closer, storages, chunkDataPacks, state, verifier, err := initStorages(chainID, protocolDataDir, chunkDataPackDir)
8484
if err != nil {
@@ -99,12 +99,94 @@ func VerifyRange(
9999
return fmt.Errorf("cannot verify blocks before the root block, from: %d, root: %d", from, root)
100100
}
101101

102-
for height := from; height <= to; height++ {
103-
log.Info().Uint64("height", height).Msg("verifying height")
104-
err := verifyHeight(height, storages.Headers, chunkDataPacks, storages.Results, state, verifier)
105-
if err != nil {
106-
return fmt.Errorf("could not verify height %d: %w", height, err)
102+
err = verifyConcurrently(from, to, nWorker, storages.Headers, chunkDataPacks, storages.Results, state, verifier, verifyHeight)
103+
if err != nil {
104+
return err
105+
}
106+
107+
return nil
108+
}
109+
110+
func verifyConcurrently(
111+
from, to uint64,
112+
nWorker uint,
113+
headers storage.Headers,
114+
chunkDataPacks storage.ChunkDataPacks,
115+
results storage.ExecutionResults,
116+
state protocol.State,
117+
verifier module.ChunkVerifier,
118+
verifyHeight func(uint64, storage.Headers, storage.ChunkDataPacks, storage.ExecutionResults, protocol.State, module.ChunkVerifier) error,
119+
) error {
120+
tasks := make(chan uint64, int(nWorker))
121+
ctx, cancel := context.WithCancel(context.Background())
122+
defer cancel() // Ensure cancel is called to release resources
123+
124+
var lowestErr error
125+
var lowestErrHeight uint64 = ^uint64(0) // Initialize to max value of uint64
126+
var mu sync.Mutex // To protect access to lowestErr and lowestErrHeight
127+
128+
// Worker function
129+
worker := func() {
130+
for {
131+
select {
132+
case <-ctx.Done():
133+
return // Stop processing tasks if context is canceled
134+
case height, ok := <-tasks:
135+
if !ok {
136+
return // Exit if the tasks channel is closed
137+
}
138+
log.Info().Uint64("height", height).Msg("verifying height")
139+
err := verifyHeight(height, headers, chunkDataPacks, results, state, verifier)
140+
if err != nil {
141+
log.Error().Uint64("height", height).Err(err).Msg("error encountered while verifying height")
142+
143+
// when encountered an error, the error might not be from the lowest height that had
144+
// error, so we need to first cancel the context to stop worker from processing further tasks
145+
// and wait until all workers are done, which will ensure all the heights before this height
146+
// that had error are processed. Then we can safely update the lowestErr and lowestErrHeight
147+
mu.Lock()
148+
if height < lowestErrHeight {
149+
lowestErr = err
150+
lowestErrHeight = height
151+
cancel() // Cancel context to stop further task dispatch
152+
}
153+
mu.Unlock()
154+
} else {
155+
log.Info().Uint64("height", height).Msg("verified height successfully")
156+
}
157+
}
158+
}
159+
}
160+
161+
// Start nWorker workers
162+
var wg sync.WaitGroup
163+
for i := 0; i < int(nWorker); i++ {
164+
wg.Add(1)
165+
go func() {
166+
defer wg.Done()
167+
worker()
168+
}()
169+
}
170+
171+
// Send tasks to workers
172+
go func() {
173+
defer close(tasks) // Close tasks channel once all tasks are pushed
174+
for height := from; height <= to; height++ {
175+
select {
176+
case <-ctx.Done():
177+
return // Stop pushing tasks if context is canceled
178+
case tasks <- height:
179+
}
107180
}
181+
}()
182+
183+
// Wait for all workers to complete
184+
wg.Wait()
185+
186+
// Check if there was an error
187+
if lowestErr != nil {
188+
log.Error().Uint64("height", lowestErrHeight).Err(lowestErr).Msg("error encountered while verifying height")
189+
return fmt.Errorf("could not verify height %d: %w", lowestErrHeight, lowestErr)
108190
}
109191

110192
return nil

‎engine/verification/verifier/verifiers_test.go‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
package verifier
2+
3+
import (
4+
"errors"
5+
"fmt"
6+
"testing"
7+
8+
"github.com/onflow/flow-go/module"
9+
mockmodule "github.com/onflow/flow-go/module/mock"
10+
"github.com/onflow/flow-go/state/protocol"
11+
"github.com/onflow/flow-go/storage"
12+
"github.com/onflow/flow-go/storage/mock"
13+
unittestMocks "github.com/onflow/flow-go/utils/unittest/mocks"
14+
)
15+
16+
func TestVerifyConcurrently(t *testing.T) {
17+
18+
tests := []struct {
19+
name string
20+
from uint64
21+
to uint64
22+
nWorker uint
23+
errors map[uint64]error // Map of heights to errors
24+
expectedErr error
25+
}{
26+
{
27+
name: "All heights verified successfully",
28+
from: 1,
29+
to: 5,
30+
nWorker: 3,
31+
errors: nil,
32+
expectedErr: nil,
33+
},
34+
{
35+
name: "Single error at a height",
36+
from: 1,
37+
to: 5,
38+
nWorker: 3,
39+
errors: map[uint64]error{3: errors.New("mock error")},
40+
expectedErr: fmt.Errorf("mock error"),
41+
},
42+
{
43+
name: "Multiple errors, lowest height returned",
44+
from: 1,
45+
to: 5,
46+
nWorker: 3,
47+
errors: map[uint64]error{2: errors.New("error 2"), 4: errors.New("error 4")},
48+
expectedErr: fmt.Errorf("error 2"),
49+
},
50+
}
51+
52+
for _, tt := range tests {
53+
t.Run(tt.name, func(t *testing.T) {
54+
// Reset mockVerifyHeight for each test
55+
mockVerifyHeight := func(
56+
height uint64,
57+
headers storage.Headers,
58+
chunkDataPacks storage.ChunkDataPacks,
59+
results storage.ExecutionResults,
60+
state protocol.State,
61+
verifier module.ChunkVerifier,
62+
) error {
63+
if err, ok := tt.errors[height]; ok {
64+
return err
65+
}
66+
return nil
67+
}
68+
69+
mockHeaders := mock.NewHeaders(t)
70+
mockChunkDataPacks := mock.NewChunkDataPacks(t)
71+
mockResults := mock.NewExecutionResults(t)
72+
mockState := unittestMocks.NewProtocolState()
73+
mockVerifier := mockmodule.NewChunkVerifier(t)
74+
75+
err := verifyConcurrently(tt.from, tt.to, tt.nWorker, mockHeaders, mockChunkDataPacks, mockResults, mockState, mockVerifier, mockVerifyHeight)
76+
if tt.expectedErr != nil {
77+
if err == nil || errors.Is(err, tt.expectedErr) {
78+
t.Fatalf("expected error: %v, got: %v", tt.expectedErr, err)
79+
}
80+
} else if err != nil {
81+
t.Fatalf("expected no error, got: %v", err)
82+
}
83+
})
84+
}
85+
}

0 commit comments

Comments
 (0)