Skip to content

Commit 7b4ef34

Browse files
prathamesh0prathamesh0
authored
Update indexer to include block hash in receipts and logs (#256)
* Update indexer to include block hash in receipts and logs * Upgrade ipld-eth-db image in docker-compose to run tests
1 parent 558a187 commit 7b4ef34

File tree

11 files changed

+37
-25
lines changed

11 files changed

+37
-25
lines changed

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ services:
55
restart: on-failure
66
depends_on:
77
- ipld-eth-db
8-
image: vulcanize/ipld-eth-db:v4.1.4-alpha
8+
image: vulcanize/ipld-eth-db:v4.2.0-alpha
99
environment:
1010
DATABASE_USER: "vdbm"
1111
DATABASE_NAME: "vulcanize_testing"

‎statediff/indexer/database/dump/indexer.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
323323

324324
rctModel := &models.ReceiptModel{
325325
BlockNumber: args.blockNumber.String(),
326+
HeaderID: args.headerID,
326327
TxID: trxID,
327328
Contract: contract,
328329
ContractHash: contractHash,
@@ -353,6 +354,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
353354

354355
logDataSet[idx] = &models.LogsModel{
355356
BlockNumber: args.blockNumber.String(),
357+
HeaderID: args.headerID,
356358
ReceiptID: trxID,
357359
Address: l.Address.String(),
358360
Index: int64(l.Index),

‎statediff/indexer/database/file/csv_writer.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ func (csw *CSVWriter) upsertAccessListElement(accessListElement models.AccessLis
281281

282282
func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
283283
var values []interface{}
284-
values = append(values, rct.BlockNumber, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
284+
values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
285285
rct.PostState, rct.PostStatus, rct.LogRoot)
286286
csw.rows <- tableRow{types.TableReceipt, values}
287287
indexerMetrics.receipts.Inc(1)
@@ -290,7 +290,7 @@ func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
290290
func (csw *CSVWriter) upsertLogCID(logs []*models.LogsModel) {
291291
for _, l := range logs {
292292
var values []interface{}
293-
values = append(values, l.BlockNumber, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
293+
values = append(values, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
294294
l.Topic1, l.Topic2, l.Topic3, l.Data)
295295
csw.rows <- tableRow{types.TableLog, values}
296296
indexerMetrics.logs.Inc(1)

‎statediff/indexer/database/file/indexer.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
371371

372372
rctModel := &models.ReceiptModel{
373373
BlockNumber: args.blockNumber.String(),
374+
HeaderID: args.headerID,
374375
TxID: txID,
375376
Contract: contract,
376377
ContractHash: contractHash,
@@ -399,6 +400,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
399400

400401
logDataSet[idx] = &models.LogsModel{
401402
BlockNumber: args.blockNumber.String(),
403+
HeaderID: args.headerID,
402404
ReceiptID: txID,
403405
Address: l.Address.String(),
404406
Index: int64(l.Index),

‎statediff/indexer/database/file/sql_writer.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -155,11 +155,11 @@ const (
155155
alInsert = "INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES " +
156156
"('%s', '%s', %d, '%s', '%s');\n"
157157

158-
rctInsert = "INSERT INTO eth.receipt_cids (block_number, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " +
159-
"post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n"
158+
rctInsert = "INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " +
159+
"post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n"
160160

161-
logInsert = "INSERT INTO eth.log_cids (block_number, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " +
162-
"topic3, log_data) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n"
161+
logInsert = "INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " +
162+
"topic3, log_data) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n"
163163

164164
stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) " +
165165
"VALUES ('%s', '%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
@@ -235,14 +235,14 @@ func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessLis
235235
}
236236

237237
func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
238-
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
238+
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
239239
rct.PostState, rct.PostStatus, rct.LogRoot))
240240
indexerMetrics.receipts.Inc(1)
241241
}
242242

243243
func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
244244
for _, l := range logs {
245-
sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.BlockNumber, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
245+
sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.BlockNumber, l.HeaderID, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
246246
l.Topic1, l.Topic2, l.Topic3, l.Data))
247247
indexerMetrics.logs.Inc(1)
248248
}

‎statediff/indexer/database/file/types/schema.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ var TableReceipt = Table{
132132
"eth.receipt_cids",
133133
[]column{
134134
{name: "block_number", dbType: bigint},
135+
{name: "header_id", dbType: varchar},
135136
{name: "tx_id", dbType: varchar},
136137
{name: "leaf_cid", dbType: text},
137138
{name: "contract", dbType: varchar},
@@ -147,6 +148,7 @@ var TableLog = Table{
147148
"eth.log_cids",
148149
[]column{
149150
{name: "block_number", dbType: bigint},
151+
{name: "header_id", dbType: varchar},
150152
{name: "leaf_cid", dbType: text},
151153
{name: "leaf_mh_key", dbType: text},
152154
{name: "rct_id", dbType: varchar},

‎statediff/indexer/database/sql/indexer.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
381381

382382
rctModel := &models.ReceiptModel{
383383
BlockNumber: args.blockNumber.String(),
384+
HeaderID: args.headerID,
384385
TxID: txID,
385386
Contract: contract,
386387
ContractHash: contractHash,
@@ -412,6 +413,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
412413

413414
logDataSet[idx] = &models.LogsModel{
414415
BlockNumber: args.blockNumber.String(),
416+
HeaderID: args.headerID,
415417
ReceiptID: txID,
416418
Address: l.Address.String(),
417419
Index: int64(l.Index),

‎statediff/indexer/database/sql/postgres/database.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func (db *DB) InsertUncleStm() string {
5252
// InsertTxStm satisfies the sql.Statements interface
5353
func (db *DB) InsertTxStm() string {
5454
return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
55-
ON CONFLICT (tx_hash, block_number) DO NOTHING`
55+
ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING`
5656
}
5757

5858
// InsertAccessListElementStm satisfies the sql.Statements interface
@@ -63,14 +63,14 @@ func (db *DB) InsertAccessListElementStm() string {
6363

6464
// InsertRctStm satisfies the sql.Statements interface
6565
func (db *DB) InsertRctStm() string {
66-
return `INSERT INTO eth.receipt_cids (block_number, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
67-
ON CONFLICT (tx_id, block_number) DO NOTHING`
66+
return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
67+
ON CONFLICT (tx_id, header_id, block_number) DO NOTHING`
6868
}
6969

7070
// InsertLogStm satisfies the sql.Statements interface
7171
func (db *DB) InsertLogStm() string {
72-
return `INSERT INTO eth.log_cids (block_number, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
73-
ON CONFLICT (rct_id, index, block_number) DO NOTHING`
72+
return `INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
73+
ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING`
7474
}
7575

7676
// InsertStateStm satisfies the sql.Statements interface

‎statediff/indexer/database/sql/writer.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
7676

7777
/*
7878
INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
79-
ON CONFLICT (tx_hash, block_number) DO NOTHING
79+
ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING
8080
*/
8181
func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
8282
_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
@@ -105,12 +105,12 @@ func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessL
105105
}
106106

107107
/*
108-
INSERT INTO eth.receipt_cids (block_number, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
109-
ON CONFLICT (tx_id, block_number) DO NOTHING
108+
INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
109+
ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
110110
*/
111111
func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
112112
_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
113-
rct.BlockNumber, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState,
113+
rct.BlockNumber, rct.HeaderID, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState,
114114
rct.PostStatus, rct.LogRoot)
115115
if err != nil {
116116
return fmt.Errorf("error upserting receipt_cids entry: %w", err)
@@ -120,13 +120,13 @@ func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
120120
}
121121

122122
/*
123-
INSERT INTO eth.log_cids (block_number, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
124-
ON CONFLICT (rct_id, index, block_number) DO NOTHING
123+
INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
124+
ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING
125125
*/
126126
func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
127127
for _, log := range logs {
128128
_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
129-
log.BlockNumber, log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1,
129+
log.BlockNumber, log.HeaderID, log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1,
130130
log.Topic2, log.Topic3, log.Data)
131131
if err != nil {
132132
return fmt.Errorf("error upserting logs entry: %w", err)

‎statediff/indexer/models/batch.go‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ type UncleBatch struct {
3939
// TxBatch holds the arguments for a batch insert of tx data
4040
type TxBatch struct {
4141
BlockNumbers []string
42-
HeaderID string
42+
HeaderIDs []string
4343
Indexes []int64
4444
TxHashes []string
4545
CIDs []string
@@ -62,6 +62,7 @@ type AccessListBatch struct {
6262
// ReceiptBatch holds the arguments for a batch insert of receipt data
6363
type ReceiptBatch struct {
6464
BlockNumbers []string
65+
HeaderIDs []string
6566
TxIDs []string
6667
LeafCIDs []string
6768
LeafMhKeys []string
@@ -75,6 +76,7 @@ type ReceiptBatch struct {
7576
// LogBatch holds the arguments for a batch insert of log data
7677
type LogBatch struct {
7778
BlockNumbers []string
79+
HeaderIDs []string
7880
LeafCIDs []string
7981
LeafMhKeys []string
8082
ReceiptIDs []string
@@ -90,7 +92,7 @@ type LogBatch struct {
9092
// StateBatch holds the arguments for a batch insert of state data
9193
type StateBatch struct {
9294
BlockNumbers []string
93-
HeaderID string
95+
HeaderIDs []string
9496
Paths [][]byte
9597
StateKeys []string
9698
NodeTypes []int
@@ -102,7 +104,7 @@ type StateBatch struct {
102104
// AccountBatch holds the arguments for a batch insert of account data
103105
type AccountBatch struct {
104106
BlockNumbers []string
105-
HeaderID string
107+
HeaderIDs []string
106108
StatePaths [][]byte
107109
Balances []string
108110
Nonces []uint64
@@ -113,7 +115,7 @@ type AccountBatch struct {
113115
// StorageBatch holds the arguments for a batch insert of storage data
114116
type StorageBatch struct {
115117
BlockNumbers []string
116-
HeaderID string
118+
HeaderIDs []string
117119
StatePaths [][]string
118120
Paths [][]byte
119121
StorageKeys []string

0 commit comments

Comments
 (0)