Skip to content

Commit 6b457b0

Browse files
i-nordeni-norden
committed
test harnest for arbitrary mainnet blocks and receipts
1 parent 1dd37be commit 6b457b0

File tree

5 files changed

+459
-181
lines changed

5 files changed

+459
-181
lines changed

‎statediff/indexer/database/file/mainnet_tests/indexer_test.go‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// VulcanizeDB
2+
// Copyright © 2021 Vulcanize
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package mainnet_tests
18+
19+
import (
20+
"context"
21+
"errors"
22+
"fmt"
23+
"os"
24+
"testing"
25+
26+
"github.com/ipfs/go-cid"
27+
"github.com/jmoiron/sqlx"
28+
"github.com/multiformats/go-multihash"
29+
"github.com/stretchr/testify/require"
30+
31+
"github.com/ethereum/go-ethereum/core/types"
32+
"github.com/ethereum/go-ethereum/params"
33+
"github.com/ethereum/go-ethereum/rlp"
34+
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
35+
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
36+
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
37+
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
38+
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
39+
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
40+
)
41+
42+
var (
43+
testBlock *types.Block
44+
testReceipts types.Receipts
45+
testHeaderCID cid.Cid
46+
sqlxdb *sqlx.DB
47+
err error
48+
chainConf = params.MainnetChainConfig
49+
)
50+
51+
func init() {
52+
if os.Getenv("MODE") != "statediff" {
53+
fmt.Println("Skipping statediff test")
54+
os.Exit(0)
55+
}
56+
}
57+
58+
func setup(t *testing.T) {
59+
testBlock, testReceipts, err = TestBlocksAndReceiptsFromEnv()
60+
require.NoError(t, err)
61+
headerRLP, err := rlp.EncodeToBytes(testBlock.Header())
62+
require.NoError(t, err)
63+
64+
testHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256)
65+
if _, err := os.Stat(file.TestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
66+
err := os.Remove(file.TestConfig.FilePath)
67+
require.NoError(t, err)
68+
}
69+
ind, err := file.NewStateDiffIndexer(context.Background(), chainConf, file.TestConfig)
70+
require.NoError(t, err)
71+
var tx interfaces.Batch
72+
tx, err = ind.PushBlock(
73+
testBlock,
74+
testReceipts,
75+
testBlock.Difficulty())
76+
require.NoError(t, err)
77+
78+
defer func() {
79+
if err := tx.Submit(err); err != nil {
80+
t.Fatal(err)
81+
}
82+
if err := ind.Close(); err != nil {
83+
t.Fatal(err)
84+
}
85+
}()
86+
for _, node := range mocks.StateDiffs {
87+
err = ind.PushStateNode(tx, node, testBlock.Hash().String())
88+
require.NoError(t, err)
89+
}
90+
91+
test_helpers.ExpectEqual(t, tx.(*file.BatchTx).BlockNumber, testBlock.Number().Uint64())
92+
93+
connStr := postgres.DefaultConfig.DbConnectionString()
94+
95+
sqlxdb, err = sqlx.Connect("postgres", connStr)
96+
if err != nil {
97+
t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err)
98+
}
99+
}
100+
101+
func dumpData(t *testing.T) {
102+
sqlFileBytes, err := os.ReadFile(file.TestConfig.FilePath)
103+
require.NoError(t, err)
104+
105+
_, err = sqlxdb.Exec(string(sqlFileBytes))
106+
require.NoError(t, err)
107+
}
108+
109+
func tearDown(t *testing.T) {
110+
file.TearDownDB(t, sqlxdb)
111+
err := os.Remove(file.TestConfig.FilePath)
112+
require.NoError(t, err)
113+
err = sqlxdb.Close()
114+
require.NoError(t, err)
115+
}
116+
117+
func TestPushBlockAndState(t *testing.T) {
118+
t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
119+
setup(t)
120+
dumpData(t)
121+
tearDown(t)
122+
})
123+
}

‎statediff/indexer/database/file/mainnet_tests/test_helpers.go‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
// VulcanizeDB
2+
// Copyright © 2021 Vulcanize
3+
4+
// This program is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Affero General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// This program is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Affero General Public License for more details.
13+
14+
// You should have received a copy of the GNU Affero General Public License
15+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package mainnet_tests
18+
19+
import (
20+
"bufio"
21+
"context"
22+
"errors"
23+
"fmt"
24+
"math/big"
25+
"os"
26+
27+
"github.com/ethereum/go-ethereum/core/types"
28+
"github.com/ethereum/go-ethereum/ethclient"
29+
"github.com/ethereum/go-ethereum/rlp"
30+
)
31+
32+
const (
33+
defaultBlockFilePath = "./block"
34+
defaultReceiptsFilePath = "./receipts"
35+
)
36+
37+
const (
38+
TEST_RAW_URL = "TEST_RAW_URL"
39+
TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER"
40+
)
41+
42+
// TestConfig holds configuration params for mainnet tests
43+
type TestConfig struct {
44+
RawURL string
45+
BlockNumber *big.Int
46+
}
47+
48+
// DefaultTestConfig is the default TestConfig
49+
var DefaultTestConfig = TestConfig{
50+
RawURL: "http://127.0.0.1:8545",
51+
BlockNumber: big.NewInt(12914665),
52+
}
53+
54+
// TestBlocksAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config
55+
func TestBlocksAndReceiptsFromEnv() (*types.Block, types.Receipts, error) {
56+
conf := DefaultTestConfig
57+
rawURL := os.Getenv(TEST_RAW_URL)
58+
if rawURL == "" {
59+
fmt.Println("Warning: no raw url configured for statediffing mainnet tests")
60+
} else {
61+
conf.RawURL = rawURL
62+
}
63+
blockNumberStr := os.Getenv(TEST_BLOCK_NUMBER)
64+
blockNumber, ok := new(big.Int).SetString(blockNumberStr, 10)
65+
if !ok {
66+
fmt.Println("Warning: no blockNumber configured for statediffing mainnet tests")
67+
} else {
68+
conf.BlockNumber = blockNumber
69+
}
70+
return TestBlocksAndReceipts(conf)
71+
}
72+
73+
// TestBlocksAndReceipts retrieves the block and receipts for the provided test config
74+
// It first tries to load files from the local system before setting up and using an ethclient.Client to pull the data
75+
func TestBlocksAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) {
76+
var cli *ethclient.Client
77+
var err error
78+
var block *types.Block
79+
var receipts types.Receipts
80+
blockFilePath := fmt.Sprintf("%s.%s.rlp", defaultBlockFilePath, conf.BlockNumber.String())
81+
if _, err = os.Stat(blockFilePath); !errors.Is(err, os.ErrNotExist) {
82+
block, err = LoadBlockRLP(blockFilePath)
83+
if err != nil {
84+
cli, err = ethclient.Dial(conf.RawURL)
85+
if err != nil {
86+
return nil, nil, err
87+
}
88+
block, err = FetchBlock(cli, conf.BlockNumber)
89+
if err != nil {
90+
return nil, nil, err
91+
}
92+
}
93+
} else {
94+
cli, err = ethclient.Dial(conf.RawURL)
95+
if err != nil {
96+
return nil, nil, err
97+
}
98+
block, err = FetchBlock(cli, conf.BlockNumber)
99+
if err != nil {
100+
return nil, nil, err
101+
}
102+
}
103+
receiptsFilePath := fmt.Sprintf("%s.%s.enc", defaultReceiptsFilePath, conf.BlockNumber.String())
104+
if _, err = os.Stat(receiptsFilePath); !errors.Is(err, os.ErrNotExist) {
105+
receipts, err = LoadReceiptsEncoding(receiptsFilePath, len(block.Transactions()))
106+
if err != nil {
107+
if cli == nil {
108+
cli, err = ethclient.Dial(conf.RawURL)
109+
if err != nil {
110+
return nil, nil, err
111+
}
112+
}
113+
receipts, err = FetchReceipts(cli, block)
114+
if err != nil {
115+
return nil, nil, err
116+
}
117+
}
118+
} else {
119+
if cli == nil {
120+
cli, err = ethclient.Dial(conf.RawURL)
121+
if err != nil {
122+
return nil, nil, err
123+
}
124+
}
125+
receipts, err = FetchReceipts(cli, block)
126+
if err != nil {
127+
return nil, nil, err
128+
}
129+
}
130+
return block, receipts, nil
131+
}
132+
133+
// FetchBlock fetches the block at the provided height using the ethclient.Client
134+
func FetchBlock(cli *ethclient.Client, blockNumber *big.Int) (*types.Block, error) {
135+
return cli.BlockByNumber(context.Background(), blockNumber)
136+
}
137+
138+
// FetchReceipts fetches the receipts for the provided block using the ethclient.Client
139+
func FetchReceipts(cli *ethclient.Client, block *types.Block) (types.Receipts, error) {
140+
receipts := make(types.Receipts, len(block.Transactions()))
141+
for i, tx := range block.Transactions() {
142+
rct, err := cli.TransactionReceipt(context.Background(), tx.Hash())
143+
if err != nil {
144+
return nil, err
145+
}
146+
receipts[i] = rct
147+
}
148+
return receipts, nil
149+
}
150+
151+
// WriteBlockRLP writes out the RLP encoding of the block to the provided filePath
152+
func WriteBlockRLP(filePath string, block *types.Block) error {
153+
if filePath == "" {
154+
filePath = fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, block.Number().String())
155+
}
156+
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
157+
return fmt.Errorf("cannot create file, file (%s) already exists", filePath)
158+
}
159+
file, err := os.Create(filePath)
160+
if err != nil {
161+
return fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
162+
}
163+
if err := block.EncodeRLP(file); err != nil {
164+
return err
165+
}
166+
return file.Close()
167+
}
168+
169+
// LoadBlockRLP loads block from the rlp at filePath
170+
func LoadBlockRLP(filePath string) (*types.Block, error) {
171+
blockBytes, err := os.ReadFile(filePath)
172+
if err != nil {
173+
return nil, err
174+
}
175+
block := new(types.Block)
176+
return block, rlp.DecodeBytes(blockBytes, block)
177+
}
178+
179+
// LoadReceiptsEncoding loads receipts from the encoding at filePath
180+
func LoadReceiptsEncoding(filePath string, cap int) (types.Receipts, error) {
181+
file, err := os.Open(filePath)
182+
if err != nil {
183+
return nil, err
184+
}
185+
defer file.Close()
186+
scanner := bufio.NewScanner(file)
187+
receipts := make(types.Receipts, 0, cap)
188+
for scanner.Scan() {
189+
rctBinary := scanner.Bytes()
190+
rct := new(types.Receipt)
191+
if err := rct.UnmarshalBinary(rctBinary); err != nil {
192+
return nil, err
193+
}
194+
receipts = append(receipts, rct)
195+
}
196+
return receipts, nil
197+
}
198+
199+
// WriteReceiptsEncoding writes out the consensus encoding of the receipts to the provided io.WriteCloser
200+
func WriteReceiptsEncoding(filePath string, blockNumber *big.Int, receipts types.Receipts) error {
201+
if filePath == "" {
202+
filePath = fmt.Sprintf("%s_%s.enc", defaultReceiptsFilePath, blockNumber.String())
203+
}
204+
if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
205+
return fmt.Errorf("cannot create file, file (%s) already exists", filePath)
206+
}
207+
file, err := os.Create(filePath)
208+
if err != nil {
209+
return fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
210+
}
211+
for _, rct := range receipts {
212+
rctEncoding, err := rct.MarshalBinary()
213+
if err != nil {
214+
return err
215+
}
216+
if _, err := file.Write(rctEncoding); err != nil {
217+
return err
218+
}
219+
if _, err := file.Write([]byte("\n")); err != nil {
220+
return err
221+
}
222+
}
223+
return file.Close()
224+
}

0 commit comments

Comments
 (0)