Skip to content

Commit b9a82f6

Browse files
i-nordeni-norden
committed
basefee is big.Int, it won't always fit in int64
1 parent b96c26f commit b9a82f6

File tree

16 files changed

+148
-106
lines changed

16 files changed

+148
-106
lines changed

‎statediff/indexer/database/dump/indexer.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -184,10 +184,10 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
184184
func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) {
185185
tx.cacheIPLD(headerNode)
186186

187-
var baseFee *int64
187+
var baseFee *string
188188
if header.BaseFee != nil {
189-
baseFee = new(int64)
190-
*baseFee = header.BaseFee.Int64()
189+
baseFee = new(string)
190+
*baseFee = header.BaseFee.String()
191191
}
192192

193193
headerID := header.Hash().String()

‎statediff/indexer/database/file/indexer.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,10 +191,10 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
191191
func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node.Node, reward, td *big.Int) string {
192192
sdi.fileWriter.upsertIPLDNode(headerNode)
193193

194-
var baseFee *int64
194+
var baseFee *string
195195
if header.BaseFee != nil {
196-
baseFee = new(int64)
197-
*baseFee = header.BaseFee.Int64()
196+
baseFee = new(string)
197+
*baseFee = header.BaseFee.String()
198198
}
199199
headerID := header.Hash().String()
200200
sdi.fileWriter.upsertHeaderCID(models.HeaderModel{

‎statediff/indexer/database/file/indexer_legacy_test.go‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,21 @@ package file_test
1818

1919
import (
2020
"context"
21+
"errors"
2122
"os"
2223
"testing"
2324

24-
"github.com/ethereum/go-ethereum/core/types"
25-
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
26-
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
2725
"github.com/ipfs/go-cid"
2826
"github.com/jmoiron/sqlx"
29-
3027
"github.com/multiformats/go-multihash"
3128
"github.com/stretchr/testify/require"
3229

33-
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
30+
"github.com/ethereum/go-ethereum/core/types"
31+
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
3432
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
3533
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
3634
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
35+
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
3736
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
3837
)
3938

@@ -46,7 +45,10 @@ var (
4645
func setupLegacy(t *testing.T) {
4746
mockLegacyBlock = legacyData.MockBlock
4847
legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256)
49-
48+
if _, err := os.Stat(file.TestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
49+
err := os.Remove(file.TestConfig.FilePath)
50+
require.NoError(t, err)
51+
}
5052
ind, err := file.NewStateDiffIndexer(context.Background(), legacyData.Config, file.TestConfig)
5153
require.NoError(t, err)
5254
var tx interfaces.Batch
@@ -69,7 +71,7 @@ func setupLegacy(t *testing.T) {
6971
require.NoError(t, err)
7072
}
7173

72-
test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, legacyData.BlockNumber.Uint64())
74+
test_helpers.ExpectEqual(t, tx.(*file.BatchTx).BlockNumber, legacyData.BlockNumber.Uint64())
7375

7476
connStr := postgres.DefaultConfig.DbConnectionString()
7577

@@ -88,6 +90,7 @@ func dumpData(t *testing.T) {
8890
}
8991

9092
func tearDown(t *testing.T) {
93+
file.TearDownDB(t, sqlxdb)
9194
err := os.Remove(file.TestConfig.FilePath)
9295
require.NoError(t, err)
9396
err = sqlxdb.Close()
@@ -100,26 +103,24 @@ func expectTrue(t *testing.T, value bool) {
100103
}
101104
}
102105

103-
func TestFIleIndexerLegacy(t *testing.T) {
104-
t.Run("Publish and index header IPLDs in a legacy tx", func(t *testing.T) {
106+
func TestFileIndexerLegacy(t *testing.T) {
107+
t.Run("Publish and index header IPLDs", func(t *testing.T) {
105108
setupLegacy(t)
106109
dumpData(t)
107110
defer tearDown(t)
108-
pgStr := `SELECT cid, cast(td AS TEXT), cast(reward AS TEXT), block_hash, base_fee
111+
pgStr := `SELECT cid, td, reward, block_hash, base_fee
109112
FROM eth.header_cids
110113
WHERE block_number = $1`
111114
// check header was properly indexed
112115
type res struct {
113116
CID string
114117
TD string
115118
Reward string
116-
BlockHash string `db:"block_hash"`
117-
BaseFee *int64 `db:"base_fee"`
119+
BlockHash string `db:"block_hash"`
120+
BaseFee *string `db:"base_fee"`
118121
}
119122
header := new(res)
120-
121-
err = sqlxdb.QueryRow(pgStr, legacyData.BlockNumber.Uint64()).Scan(
122-
&header.CID, &header.TD, &header.Reward, &header.BlockHash, &header.BaseFee)
123+
err = sqlxdb.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header)
123124
require.NoError(t, err)
124125

125126
test_helpers.ExpectEqual(t, header.CID, legacyHeaderCID.String())

‎statediff/indexer/database/file/indexer_test.go‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,29 +19,29 @@ package file_test
1919
import (
2020
"bytes"
2121
"context"
22+
"errors"
2223
"fmt"
2324
"os"
2425
"testing"
2526

26-
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
27-
"github.com/jmoiron/sqlx"
27+
"github.com/ethereum/go-ethereum/common"
28+
"github.com/ethereum/go-ethereum/rlp"
29+
"github.com/ethereum/go-ethereum/statediff/indexer/models"
30+
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
2831

2932
"github.com/ipfs/go-cid"
3033
blockstore "github.com/ipfs/go-ipfs-blockstore"
3134
dshelp "github.com/ipfs/go-ipfs-ds-help"
35+
"github.com/jmoiron/sqlx"
3236
"github.com/multiformats/go-multihash"
3337
"github.com/stretchr/testify/require"
3438

35-
"github.com/ethereum/go-ethereum/common"
3639
"github.com/ethereum/go-ethereum/core/types"
37-
"github.com/ethereum/go-ethereum/rlp"
38-
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
40+
"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
3941
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
4042
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
4143
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
4244
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
43-
"github.com/ethereum/go-ethereum/statediff/indexer/models"
44-
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
4545
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
4646
)
4747

@@ -135,7 +135,11 @@ func init() {
135135
}
136136

137137
func setup(t *testing.T) {
138-
ind, err := file.NewStateDiffIndexer(context.Background(), legacyData.Config, file.TestConfig)
138+
if _, err := os.Stat(file.TestConfig.FilePath); !errors.Is(err, os.ErrNotExist) {
139+
err := os.Remove(file.TestConfig.FilePath)
140+
require.NoError(t, err)
141+
}
142+
ind, err = file.NewStateDiffIndexer(context.Background(), mocks.TestConfig, file.TestConfig)
139143
require.NoError(t, err)
140144
var tx interfaces.Batch
141145
tx, err = ind.PushBlock(
@@ -158,7 +162,7 @@ func setup(t *testing.T) {
158162
require.NoError(t, err)
159163
}
160164

161-
test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, mocks.BlockNumber.Uint64())
165+
test_helpers.ExpectEqual(t, tx.(*file.BatchTx).BlockNumber, mocks.BlockNumber.Uint64())
162166

163167
connStr := postgres.DefaultConfig.DbConnectionString()
164168

@@ -168,7 +172,7 @@ func setup(t *testing.T) {
168172
}
169173
}
170174

171-
func TestSQLXIndexer(t *testing.T) {
175+
func TestFileIndexer(t *testing.T) {
172176
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
173177
setup(t)
174178
dumpData(t)
@@ -181,18 +185,19 @@ func TestSQLXIndexer(t *testing.T) {
181185
CID string
182186
TD string
183187
Reward string
184-
BlockHash string `db:"block_hash"`
185-
BaseFee *int64 `db:"base_fee"`
188+
BlockHash string `db:"block_hash"`
189+
BaseFee *string `db:"base_fee"`
186190
}
187191
header := new(res)
188192
err = sqlxdb.QueryRowx(pgStr, mocks.BlockNumber.Uint64()).StructScan(header)
189193
if err != nil {
190194
t.Fatal(err)
191195
}
196+
192197
test_helpers.ExpectEqual(t, header.CID, headerCID.String())
193198
test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String())
194199
test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250")
195-
test_helpers.ExpectEqual(t, *header.BaseFee, mocks.MockHeader.BaseFee.Int64())
200+
test_helpers.ExpectEqual(t, *header.BaseFee, mocks.MockHeader.BaseFee.String())
196201
dc, err := cid.Decode(header.CID)
197202
if err != nil {
198203
t.Fatal(err)
@@ -206,7 +211,6 @@ func TestSQLXIndexer(t *testing.T) {
206211
}
207212
test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp)
208213
})
209-
210214
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
211215
setup(t)
212216
dumpData(t)

‎statediff/indexer/database/file/test_helpers.go‎

Lines changed: 58 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,64 @@
11
package file
22

3-
import "io"
3+
import (
4+
"testing"
45

5-
type MemWriter struct {
6-
bytes []byte
7-
}
8-
9-
func NewMemWriter() io.WriteCloser {
10-
return &MemWriter{}
11-
}
6+
"github.com/jmoiron/sqlx"
7+
)
128

13-
// Write satisfies io.WriteCloser
14-
func (mw *MemWriter) Write(b []byte) (int, error) {
15-
mw.bytes = append(mw.bytes, b...)
16-
return len(b), nil
17-
}
18-
19-
// Close satisfies io.WriteCloser
20-
func (mw *MemWriter) Close() error {
21-
mw.bytes = []byte{}
22-
return nil
23-
}
9+
// TearDownDB is used to tear down the watcher dbs after tests
10+
func TearDownDB(t *testing.T, db *sqlx.DB) {
11+
tx, err := db.Begin()
12+
if err != nil {
13+
t.Fatal(err)
14+
}
2415

25-
// ReadAll returns all the bytes written to the memory writer
26-
func (mw *MemWriter) ReadAll() []byte {
27-
return mw.bytes
16+
_, err = tx.Exec(`DELETE FROM eth.header_cids`)
17+
if err != nil {
18+
t.Fatal(err)
19+
}
20+
_, err = tx.Exec(`DELETE FROM eth.uncle_cids`)
21+
if err != nil {
22+
t.Fatal(err)
23+
}
24+
_, err = tx.Exec(`DELETE FROM eth.transaction_cids`)
25+
if err != nil {
26+
t.Fatal(err)
27+
}
28+
_, err = tx.Exec(`DELETE FROM eth.receipt_cids`)
29+
if err != nil {
30+
t.Fatal(err)
31+
}
32+
_, err = tx.Exec(`DELETE FROM eth.state_cids`)
33+
if err != nil {
34+
t.Fatal(err)
35+
}
36+
_, err = tx.Exec(`DELETE FROM eth.storage_cids`)
37+
if err != nil {
38+
t.Fatal(err)
39+
}
40+
_, err = tx.Exec(`DELETE FROM eth.state_accounts`)
41+
if err != nil {
42+
t.Fatal(err)
43+
}
44+
_, err = tx.Exec(`DELETE FROM eth.access_list_elements`)
45+
if err != nil {
46+
t.Fatal(err)
47+
}
48+
_, err = tx.Exec(`DELETE FROM eth.log_cids`)
49+
if err != nil {
50+
t.Fatal(err)
51+
}
52+
_, err = tx.Exec(`DELETE FROM blocks`)
53+
if err != nil {
54+
t.Fatal(err)
55+
}
56+
_, err = tx.Exec(`DELETE FROM nodes`)
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
err = tx.Commit()
61+
if err != nil {
62+
t.Fatal(err)
63+
}
2864
}

‎statediff/indexer/database/file/writer.go‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -121,38 +121,38 @@ const (
121121
nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " +
122122
"('%s', '%s', '%s', '%s', %d);\n"
123123

124-
ipldInsert = "INSERT INTO public.blocks (key, data) VALUES ('%s', '%x');\n"
124+
ipldInsert = "INSERT INTO public.blocks (key, data) VALUES ('%s', '\\x%x');\n"
125125

126126
headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, " +
127127
"state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee) VALUES " +
128-
"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%x', %d, '%s', %d, %d);\n"
128+
"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %d, %s);\n"
129129

130130
headerInsertWithoutBaseFee = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, " +
131131
"reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee) VALUES " +
132-
"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%x', %d, '%s', %d, NULL);\n"
132+
"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %d, NULL);\n"
133133

134134
uncleInsert = "INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES " +
135135
"('%s', '%s', '%s', '%s', '%s', '%s');\n"
136136

137137
txInsert = "INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type) " +
138-
"VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%x', %d);\n"
138+
"VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '\\x%x', %d);\n"
139139

140140
alInsert = "INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ('%s', %d, '%s', '%s');\n"
141141

142142
rctInsert = "INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " +
143143
"post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n"
144144

145145
logInsert = "INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " +
146-
"topic3, log_data) VALUES ('%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%x');\n"
146+
"topic3, log_data) VALUES ('%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n"
147147

148148
stateInsert = "INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) " +
149-
"VALUES ('%s', '%s', '%s', '%x', %d, %t, '%s');\n"
149+
"VALUES ('%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
150150

151151
accountInsert = "INSERT INTO eth.state_accounts (header_id, state_path, balance, nonce, code_hash, storage_root) " +
152-
"VALUES ('%s', '%x', '%s', %d, '%x', '%s');\n"
152+
"VALUES ('%s', '\\x%x', '%s', %d, '\\x%x', '%s');\n"
153153

154154
storageInsert = "INSERT INTO eth.storage_cids (header_id, state_path, storage_leaf_key, cid, storage_path, " +
155-
"node_type, diff, mh_key) VALUES ('%s', '%x', '%s', '%s', '%x', %d, %t, '%s');\n"
155+
"node_type, diff, mh_key) VALUES ('%s', '\\x%x', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
156156
)
157157

158158
func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
@@ -199,7 +199,7 @@ func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
199199
} else {
200200
stmt = fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
201201
header.TotalDifficulty, header.NodeID, header.Reward, header.StateRoot, header.TxRoot,
202-
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.BaseFee)
202+
header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, *header.BaseFee)
203203
}
204204
sqw.stmts <- []byte(stmt)
205205
indexerMetrics.blocks.Inc(1)

‎statediff/indexer/database/sql/indexer.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -235,10 +235,10 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
235235
func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, headerNode node.Node, reward, td *big.Int) (string, error) {
236236
tx.cacheIPLD(headerNode)
237237

238-
var baseFee *int64
238+
var baseFee *string
239239
if header.BaseFee != nil {
240-
baseFee = new(int64)
241-
*baseFee = header.BaseFee.Int64()
240+
baseFee = new(string)
241+
*baseFee = header.BaseFee.String()
242242
}
243243
headerID := header.Hash().String()
244244
// index header

‎statediff/indexer/database/sql/pgx_indexer_legacy_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func setupLegacyPGX(t *testing.T) {
6060
}
6161

6262
func TestPGXIndexerLegacy(t *testing.T) {
63-
t.Run("Publish and index header IPLDs in a legacy tx", func(t *testing.T) {
63+
t.Run("Publish and index header IPLDs", func(t *testing.T) {
6464
setupLegacyPGX(t)
6565
defer tearDown(t)
6666
pgStr := `SELECT cid, cast(td AS TEXT), cast(reward AS TEXT), block_hash, base_fee

0 commit comments

Comments
 (0)