Skip to content

Commit 13ab5ca

Browse files
arijitADarijitAD
authored
Merge pull request #119 from vulcanize/dedup-receipt
Dedup receipt data.
2 parents bb4ae91 + b1f2fcf commit 13ab5ca

File tree

12 files changed

+203
-101
lines changed

12 files changed

+203
-101
lines changed

‎statediff/db/migrations/00007_create_eth_receipt_cids_table.sql‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
CREATE TABLE eth.receipt_cids (
33
id SERIAL PRIMARY KEY,
44
tx_id INTEGER NOT NULL REFERENCES eth.transaction_cids (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
5-
cid TEXT NOT NULL,
6-
mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
5+
leaf_cid TEXT NOT NULL,
6+
leaf_mh_key TEXT NOT NULL REFERENCES public.blocks (key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
77
contract VARCHAR(66),
88
contract_hash VARCHAR(66),
99
post_state VARCHAR(66),

‎statediff/db/migrations/00013_create_cid_indexes.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ CREATE INDEX tx_src_index ON eth.transaction_cids USING btree (src);
2828
-- receipt indexes
2929
CREATE INDEX rct_tx_id_index ON eth.receipt_cids USING btree (tx_id);
3030

31-
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
31+
CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
3232

33-
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
33+
CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
3434

3535
CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
3636

@@ -91,8 +91,8 @@ DROP INDEX eth.state_header_id_index;
9191
-- receipt indexes
9292
DROP INDEX eth.rct_contract_hash_index;
9393
DROP INDEX eth.rct_contract_index;
94-
DROP INDEX eth.rct_mh_index;
95-
DROP INDEX eth.rct_cid_index;
94+
DROP INDEX eth.rct_leaf_mh_index;
95+
DROP INDEX eth.rct_leaf_cid_index;
9696
DROP INDEX eth.rct_tx_id_index;
9797

9898
-- transaction indexes

‎statediff/db/schema.sql‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,8 @@ ALTER SEQUENCE eth.log_cids_id_seq OWNED BY eth.log_cids.id;
363363
CREATE TABLE eth.receipt_cids (
364364
id integer NOT NULL,
365365
tx_id integer NOT NULL,
366-
cid text NOT NULL,
367-
mh_key text NOT NULL,
366+
leaf_cid text NOT NULL,
367+
leaf_mh_key text NOT NULL,
368368
contract character varying(66),
369369
contract_hash character varying(66),
370370
post_state character varying(66),
@@ -1012,10 +1012,10 @@ CREATE INDEX log_topic3_index ON eth.log_cids USING btree (topic3);
10121012

10131013

10141014
--
1015-
-- Name: rct_cid_index; Type: INDEX; Schema: eth; Owner: -
1015+
-- Name: rct_leaf_cid_index; Type: INDEX; Schema: eth; Owner: -
10161016
--
10171017

1018-
CREATE INDEX rct_cid_index ON eth.receipt_cids USING btree (cid);
1018+
CREATE INDEX rct_leaf_cid_index ON eth.receipt_cids USING btree (leaf_cid);
10191019

10201020

10211021
--
@@ -1033,10 +1033,10 @@ CREATE INDEX rct_contract_index ON eth.receipt_cids USING btree (contract);
10331033

10341034

10351035
--
1036-
-- Name: rct_mh_index; Type: INDEX; Schema: eth; Owner: -
1036+
-- Name: rct_leaf_mh_index; Type: INDEX; Schema: eth; Owner: -
10371037
--
10381038

1039-
CREATE INDEX rct_mh_index ON eth.receipt_cids USING btree (mh_key);
1039+
CREATE INDEX rct_leaf_mh_index ON eth.receipt_cids USING btree (leaf_mh_key);
10401040

10411041

10421042
--
@@ -1287,7 +1287,7 @@ ALTER TABLE ONLY eth.log_cids
12871287
--
12881288

12891289
ALTER TABLE ONLY eth.receipt_cids
1290-
ADD CONSTRAINT receipt_cids_mh_key_fkey FOREIGN KEY (mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
1290+
ADD CONSTRAINT receipt_cids_leaf_mh_key_fkey FOREIGN KEY (leaf_mh_key) REFERENCES public.blocks(key) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
12911291

12921292

12931293
--

‎statediff/indexer/indexer.go‎

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -110,13 +110,13 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
110110
}
111111

112112
// Generate the block iplds
113-
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, err := ipld.FromBlockAndReceipts(block, receipts)
113+
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld.FromBlockAndReceipts(block, receipts)
114114
if err != nil {
115115
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
116116
}
117117

118-
if len(txNodes) != len(txTrieNodes) && len(rctNodes) != len(rctTrieNodes) && len(txNodes) != len(rctNodes) {
119-
return nil, fmt.Errorf("expected number of transactions (%d), transaction trie nodes (%d), receipts (%d), and receipt trie nodes (%d)to be equal", len(txNodes), len(txTrieNodes), len(rctNodes), len(rctTrieNodes))
118+
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
119+
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d)to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
120120
}
121121

122122
// Calculate reward
@@ -202,6 +202,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
202202
txTrieNodes: txTrieNodes,
203203
logTrieNodes: logTrieNodes,
204204
logLeafNodeCIDs: logLeafNodeCIDs,
205+
rctLeafNodeCIDs: rctLeafNodeCIDs,
205206
})
206207
if err != nil {
207208
return nil, err
@@ -288,6 +289,7 @@ type processArgs struct {
288289
txTrieNodes []*ipld.EthTxTrie
289290
logTrieNodes [][]*ipld.EthLogTrie
290291
logLeafNodeCIDs [][]cid.Cid
292+
rctLeafNodeCIDs []cid.Cid
291293
}
292294

293295
// processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
@@ -309,13 +311,10 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs
309311
}
310312

311313
// publish the txs and receipts
312-
txNode, rctNode := args.txNodes[i], args.rctNodes[i]
314+
txNode := args.txNodes[i]
313315
if err := shared.PublishIPLD(tx, txNode); err != nil {
314316
return fmt.Errorf("error publishing tx IPLD: %v", err)
315317
}
316-
if err := shared.PublishIPLD(tx, rctNode); err != nil {
317-
return fmt.Errorf("error publishing rct IPLD: %v", err)
318-
}
319318

320319
// Indexing
321320
// extract topic and contract data from the receipt for indexing
@@ -390,13 +389,18 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs
390389
return err
391390
}
392391
}
392+
393393
// index the receipt
394+
if !args.rctLeafNodeCIDs[i].Defined() {
395+
return fmt.Errorf("invalid receipt leaf node cid")
396+
}
397+
394398
rctModel := &models.ReceiptModel{
395399
Contract: contract,
396400
ContractHash: contractHash,
397-
CID: rctNode.Cid().String(),
398-
MhKey: shared.MultihashKeyFromCID(rctNode.Cid()),
399-
LogRoot: rctNode.LogRoot.String(),
401+
LeafCID: args.rctLeafNodeCIDs[i].String(),
402+
LeafMhKey: shared.MultihashKeyFromCID(args.rctLeafNodeCIDs[i]),
403+
LogRoot: args.rctNodes[i].LogRoot.String(),
400404
}
401405
if len(receipt.PostState) == 0 {
402406
rctModel.PostStatus = receipt.Status

‎statediff/indexer/indexer_test.go‎

Lines changed: 65 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"github.com/ethereum/go-ethereum/core/types"
2727
"github.com/ethereum/go-ethereum/rlp"
2828
"github.com/ethereum/go-ethereum/statediff/indexer"
29+
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
2930
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
3031
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
3132
"github.com/ethereum/go-ethereum/statediff/indexer/models"
@@ -310,62 +311,94 @@ func TestPublishAndIndexer(t *testing.T) {
310311
}
311312
})
312313

313-
t.Run("Publish and index log IPLDs for single receipt", func(t *testing.T) {
314+
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
314315
setup(t)
315316
defer tearDown(t)
317+
318+
rcts := make([]string, 0)
319+
pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
320+
WHERE receipt_cids.tx_id = transaction_cids.id
321+
AND transaction_cids.header_id = header_cids.id
322+
AND header_cids.block_number = $1
323+
ORDER BY transaction_cids.index`
324+
err = db.Select(&rcts, pgStr, mocks.BlockNumber.Uint64())
325+
if err != nil {
326+
t.Fatal(err)
327+
}
328+
316329
type logIPLD struct {
317330
Index int `db:"index"`
318331
Address string `db:"address"`
319332
Data []byte `db:"data"`
320333
Topic0 string `db:"topic0"`
321334
Topic1 string `db:"topic1"`
322335
}
323-
324-
results := make([]logIPLD, 0)
325-
pgStr := `SELECT log_cids.index, log_cids.address, log_cids.Topic0, log_cids.Topic1, data FROM eth.log_cids
336+
for i := range rcts {
337+
results := make([]logIPLD, 0)
338+
pgStr = `SELECT log_cids.index, log_cids.address, log_cids.Topic0, log_cids.Topic1, data FROM eth.log_cids
326339
INNER JOIN eth.receipt_cids ON (log_cids.receipt_id = receipt_cids.id)
327340
INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key)
328-
WHERE receipt_cids.cid = $1 ORDER BY eth.log_cids.index ASC`
329-
err = db.Select(&results, pgStr, rct4CID.String())
330-
require.NoError(t, err)
341+
WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC`
342+
err = db.Select(&results, pgStr, rcts[i])
343+
require.NoError(t, err)
331344

332-
// expecting MockLog1 and MockLog2 for mockReceipt4
333-
expectedLogs := mocks.MockReceipts[3].Logs
334-
shared.ExpectEqual(t, len(results), len(expectedLogs))
345+
// expecting MockLog1 and MockLog2 for mockReceipt4
346+
expectedLogs := mocks.MockReceipts[i].Logs
347+
shared.ExpectEqual(t, len(results), len(expectedLogs))
335348

336-
var nodeElements []interface{}
337-
for idx, r := range results {
338-
// Decode the log leaf node.
339-
err = rlp.DecodeBytes(r.Data, &nodeElements)
340-
require.NoError(t, err)
349+
var nodeElements []interface{}
350+
for idx, r := range results {
351+
// Decode the log leaf node.
352+
err = rlp.DecodeBytes(r.Data, &nodeElements)
353+
require.NoError(t, err)
341354

342-
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
343-
require.NoError(t, err)
355+
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
356+
require.NoError(t, err)
344357

345-
// 2nd element of the leaf node contains the encoded log data.
346-
shared.ExpectEqual(t, logRaw, nodeElements[1].([]byte))
358+
// 2nd element of the leaf node contains the encoded log data.
359+
shared.ExpectEqual(t, logRaw, nodeElements[1].([]byte))
360+
}
347361
}
348362
})
349363

350364
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
351365
setup(t)
352366
defer tearDown(t)
367+
353368
// check receipts were properly indexed
354369
rcts := make([]string, 0)
355-
pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
370+
pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
356371
WHERE receipt_cids.tx_id = transaction_cids.id
357372
AND transaction_cids.header_id = header_cids.id
358-
AND header_cids.block_number = $1`
373+
AND header_cids.block_number = $1 order by transaction_cids.id`
359374
err = db.Select(&rcts, pgStr, mocks.BlockNumber.Uint64())
360375
if err != nil {
361376
t.Fatal(err)
362377
}
363378
shared.ExpectEqual(t, len(rcts), 5)
364-
expectTrue(t, shared.ListContainsString(rcts, rct1CID.String()))
365-
expectTrue(t, shared.ListContainsString(rcts, rct2CID.String()))
366-
expectTrue(t, shared.ListContainsString(rcts, rct3CID.String()))
367-
expectTrue(t, shared.ListContainsString(rcts, rct4CID.String()))
368-
expectTrue(t, shared.ListContainsString(rcts, rct5CID.String()))
379+
380+
for idx, rctLeafCID := range rcts {
381+
result := make([]ipfs.BlockModel, 0)
382+
pgStr = `SELECT data
383+
FROM eth.receipt_cids
384+
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
385+
WHERE receipt_cids.leaf_cid = $1`
386+
err = db.Select(&result, pgStr, rctLeafCID)
387+
if err != nil {
388+
t.Fatal(err)
389+
}
390+
391+
// Decode the log leaf node.
392+
var nodeElements []interface{}
393+
err = rlp.DecodeBytes(result[0].Data, &nodeElements)
394+
require.NoError(t, err)
395+
396+
expectedRct, err := mocks.MockReceipts[idx].MarshalBinary()
397+
require.NoError(t, err)
398+
399+
shared.ExpectEqual(t, expectedRct, nodeElements[1].([]byte))
400+
}
401+
369402
// and published
370403
for _, c := range rcts {
371404
dc, err := cid.Decode(c)
@@ -379,11 +412,12 @@ func TestPublishAndIndexer(t *testing.T) {
379412
if err != nil {
380413
t.Fatal(err)
381414
}
415+
382416
switch c {
383417
case rct1CID.String():
384418
shared.ExpectEqual(t, data, rct1)
385419
var postStatus uint64
386-
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE cid = $1`
420+
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
387421
err = db.Get(&postStatus, pgStr, c)
388422
if err != nil {
389423
t.Fatal(err)
@@ -392,7 +426,7 @@ func TestPublishAndIndexer(t *testing.T) {
392426
case rct2CID.String():
393427
shared.ExpectEqual(t, data, rct2)
394428
var postState string
395-
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1`
429+
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
396430
err = db.Get(&postState, pgStr, c)
397431
if err != nil {
398432
t.Fatal(err)
@@ -401,7 +435,7 @@ func TestPublishAndIndexer(t *testing.T) {
401435
case rct3CID.String():
402436
shared.ExpectEqual(t, data, rct3)
403437
var postState string
404-
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1`
438+
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
405439
err = db.Get(&postState, pgStr, c)
406440
if err != nil {
407441
t.Fatal(err)
@@ -410,7 +444,7 @@ func TestPublishAndIndexer(t *testing.T) {
410444
case rct4CID.String():
411445
shared.ExpectEqual(t, data, rct4)
412446
var postState string
413-
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1`
447+
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
414448
err = db.Get(&postState, pgStr, c)
415449
if err != nil {
416450
t.Fatal(err)
@@ -419,7 +453,7 @@ func TestPublishAndIndexer(t *testing.T) {
419453
case rct5CID.String():
420454
shared.ExpectEqual(t, data, rct5)
421455
var postState string
422-
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1`
456+
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
423457
err = db.Get(&postState, pgStr, c)
424458
if err != nil {
425459
t.Fatal(err)

0 commit comments

Comments
 (0)