Skip to content

Commit af67b5b

Browse files
n8maningern8maninger
authored
Merge pull request #261 from SiaFoundation/christopher/clean-up-indexing-more
Remove transaction ID maps
2 parents 0dbe60d + 894642a commit af67b5b

File tree

2 files changed

+192
-105
lines changed

2 files changed

+192
-105
lines changed

‎persist/sqlite/consensus.go‎

Lines changed: 109 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -44,153 +44,192 @@ func addMinerPayouts(tx *txn, bid types.BlockID, scos []types.SiacoinOutput) err
4444
return nil
4545
}
4646

47-
func addMinerFees(tx *txn, id int64, txn types.Transaction) error {
47+
func addMinerFees(tx *txn, dbID int64, minerFees []types.Currency) error {
48+
if len(minerFees) == 0 {
49+
return nil
50+
}
51+
4852
stmt, err := tx.Prepare(`INSERT INTO transaction_miner_fees(transaction_id, transaction_order, fee) VALUES (?, ?, ?)`)
4953
if err != nil {
5054
return fmt.Errorf("addMinerFees: failed to prepare statement: %w", err)
5155
}
5256
defer stmt.Close()
5357

54-
for i, fee := range txn.MinerFees {
55-
if _, err := stmt.Exec(id, i, encode(fee)); err != nil {
58+
for i, fee := range minerFees {
59+
if _, err := stmt.Exec(dbID, i, encode(fee)); err != nil {
5660
return fmt.Errorf("addMinerFees: failed to execute statement: %w", err)
5761
}
5862
}
5963
return nil
6064
}
6165

62-
func addArbitraryData(tx *txn, id int64, txn types.Transaction) error {
66+
func addArbitraryData(tx *txn, dbID int64, arbitraryData [][]byte) error {
67+
if len(arbitraryData) == 0 {
68+
return nil
69+
}
70+
6371
stmt, err := tx.Prepare(`INSERT INTO transaction_arbitrary_data(transaction_id, transaction_order, data) VALUES (?, ?, ?)`)
6472
if err != nil {
6573
return fmt.Errorf("addArbitraryData: failed to prepare statement: %w", err)
6674
}
6775
defer stmt.Close()
6876

69-
for i, arbitraryData := range txn.ArbitraryData {
70-
if _, err := stmt.Exec(id, i, arbitraryData); err != nil {
77+
for i, arb := range arbitraryData {
78+
if _, err := stmt.Exec(dbID, i, arb); err != nil {
7179
return fmt.Errorf("addArbitraryData: failed to execute statement: %w", err)
7280
}
7381
}
7482
return nil
7583
}
7684

77-
func addSignatures(tx *txn, id int64, txn types.Transaction) error {
85+
func addSignatures(tx *txn, dbID int64, signatures []types.TransactionSignature) error {
86+
if len(signatures) == 0 {
87+
return nil
88+
}
89+
7890
stmt, err := tx.Prepare(`INSERT INTO transaction_signatures(transaction_id, transaction_order, parent_id, public_key_index, timelock, covered_fields, signature) VALUES (?, ?, ?, ?, ?, ?, ?)`)
7991
if err != nil {
80-
return fmt.Errorf("addMinerFees: failed to prepare statement: %w", err)
92+
return fmt.Errorf("addSignatures: failed to prepare statement: %w", err)
8193
}
8294
defer stmt.Close()
8395

84-
for i, sig := range txn.Signatures {
85-
if _, err := stmt.Exec(id, i, encode(sig.ParentID), sig.PublicKeyIndex, encode(sig.Timelock), encode(sig.CoveredFields), sig.Signature); err != nil {
86-
return fmt.Errorf("addMinerFees: failed to execute statement: %w", err)
96+
for i, sig := range signatures {
97+
if _, err := stmt.Exec(dbID, i, encode(sig.ParentID), sig.PublicKeyIndex, encode(sig.Timelock), encode(sig.CoveredFields), sig.Signature); err != nil {
98+
return fmt.Errorf("addSignatures: failed to execute statement: %w", err)
8799
}
88100
}
89101
return nil
90102
}
91103

92-
func addSiacoinInputs(tx *txn, id int64, txn types.Transaction) error {
104+
func addSiacoinInputs(tx *txn, dbID int64, siacoinInputs []types.SiacoinInput) error {
105+
if len(siacoinInputs) == 0 {
106+
return nil
107+
}
108+
93109
stmt, err := tx.Prepare(`INSERT INTO transaction_siacoin_inputs(transaction_id, transaction_order, unlock_conditions, parent_id) VALUES (?, ?, ?, (SELECT id FROM siacoin_elements WHERE output_id = ?))`)
94110
if err != nil {
95111
return fmt.Errorf("addSiacoinInputs: failed to prepare statement: %w", err)
96112
}
97113
defer stmt.Close()
98114

99-
for i, sci := range txn.SiacoinInputs {
100-
if _, err := stmt.Exec(id, i, encode(sci.UnlockConditions), encode(sci.ParentID)); err != nil {
115+
for i, sci := range siacoinInputs {
116+
if _, err := stmt.Exec(dbID, i, encode(sci.UnlockConditions), encode(sci.ParentID)); err != nil {
101117
return fmt.Errorf("addSiacoinInputs: failed to execute statement: %w", err)
102118
}
103119
}
104120
return nil
105121
}
106122

107-
func addSiacoinOutputs(tx *txn, id int64, txn types.Transaction) error {
123+
func addSiacoinOutputs(tx *txn, dbID int64, txn types.Transaction) error {
124+
if len(txn.SiacoinOutputs) == 0 {
125+
return nil
126+
}
127+
108128
stmt, err := tx.Prepare(`INSERT INTO transaction_siacoin_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, (SELECT id FROM siacoin_elements WHERE output_id = ?))`)
109129
if err != nil {
110130
return fmt.Errorf("addSiacoinOutputs: failed to prepare statement: %w", err)
111131
}
112132
defer stmt.Close()
113133

114134
for i := range txn.SiacoinOutputs {
115-
if _, err := stmt.Exec(id, i, encode(txn.SiacoinOutputID(i))); err != nil {
135+
if _, err := stmt.Exec(dbID, i, encode(txn.SiacoinOutputID(i))); err != nil {
116136
return fmt.Errorf("addSiacoinOutputs: failed to execute statement: %w", err)
117137
}
118138
}
119139
return nil
120140
}
121141

122-
func addSiafundInputs(tx *txn, id int64, txn types.Transaction) error {
142+
func addSiafundInputs(tx *txn, dbID int64, siafundInputs []types.SiafundInput) error {
143+
if len(siafundInputs) == 0 {
144+
return nil
145+
}
146+
123147
stmt, err := tx.Prepare(`INSERT INTO transaction_siafund_inputs(transaction_id, transaction_order, unlock_conditions, claim_address, parent_id) VALUES (?, ?, ?, ?, (SELECT id FROM siafund_elements WHERE output_id = ?))`)
124148
if err != nil {
125149
return fmt.Errorf("addSiafundInputs: failed to prepare statement: %w", err)
126150
}
127151
defer stmt.Close()
128152

129-
for i, sfi := range txn.SiafundInputs {
130-
if _, err := stmt.Exec(id, i, encode(sfi.UnlockConditions), encode(sfi.ClaimAddress), encode(sfi.ParentID)); err != nil {
153+
for i, sfi := range siafundInputs {
154+
if _, err := stmt.Exec(dbID, i, encode(sfi.UnlockConditions), encode(sfi.ClaimAddress), encode(sfi.ParentID)); err != nil {
131155
return fmt.Errorf("addSiafundInputs: failed to execute statement: %w", err)
132156
}
133157
}
134158

135159
return nil
136160
}
137161

138-
func addSiafundOutputs(tx *txn, id int64, txn types.Transaction) error {
162+
func addSiafundOutputs(tx *txn, dbID int64, txn types.Transaction) error {
163+
if len(txn.SiafundOutputs) == 0 {
164+
return nil
165+
}
166+
139167
stmt, err := tx.Prepare(`INSERT INTO transaction_siafund_outputs(transaction_id, transaction_order, output_id) VALUES (?, ?, (SELECT id FROM siafund_elements WHERE output_id = ?))`)
140168
if err != nil {
141169
return fmt.Errorf("addSiafundOutputs: failed to prepare statement: %w", err)
142170
}
143171
defer stmt.Close()
144172

145173
for i := range txn.SiafundOutputs {
146-
if _, err := stmt.Exec(id, i, encode(txn.SiafundOutputID(i))); err != nil {
174+
if _, err := stmt.Exec(dbID, i, encode(txn.SiafundOutputID(i))); err != nil {
147175
return fmt.Errorf("addSiafundOutputs: failed to execute statement: %w", err)
148176
}
149177
}
150178
return nil
151179
}
152180

153-
func addFileContracts(tx *txn, id int64, txn types.Transaction) error {
181+
func addFileContracts(tx *txn, dbID int64, txn types.Transaction) error {
182+
if len(txn.FileContracts) == 0 {
183+
return nil
184+
}
185+
154186
stmt, err := tx.Prepare(`INSERT INTO transaction_file_contracts(transaction_id, transaction_order, contract_id) VALUES (?, ?, (SELECT id FROM file_contract_elements WHERE contract_id = ? AND revision_number = ?))`)
155187
if err != nil {
156188
return fmt.Errorf("addFileContracts: failed to prepare statement: %w", err)
157189
}
158190
defer stmt.Close()
159191

160192
for i, fc := range txn.FileContracts {
161-
if _, err := stmt.Exec(id, i, encode(txn.FileContractID(i)), encode(fc.RevisionNumber)); err != nil {
193+
if _, err := stmt.Exec(dbID, i, encode(txn.FileContractID(i)), encode(fc.RevisionNumber)); err != nil {
162194
return fmt.Errorf("addFileContracts: failed to execute transaction_file_contracts statement: %w", err)
163195
}
164196
}
165197
return nil
166198
}
167199

168-
func addFileContractRevisions(tx *txn, id int64, txn types.Transaction) error {
200+
func addFileContractRevisions(tx *txn, dbID int64, fileContractRevisions []types.FileContractRevision) error {
201+
if len(fileContractRevisions) == 0 {
202+
return nil
203+
}
204+
169205
stmt, err := tx.Prepare(`INSERT INTO transaction_file_contract_revisions(transaction_id, transaction_order, parent_id, unlock_conditions, contract_id) VALUES (?, ?, ?, ?, (SELECT id FROM file_contract_elements WHERE contract_id = ? AND revision_number = ?))`)
170206
if err != nil {
171207
return fmt.Errorf("addFileContractRevisions: failed to prepare statement: %w", err)
172208
}
173209
defer stmt.Close()
174210

175-
for i := range txn.FileContractRevisions {
176-
fcr := &txn.FileContractRevisions[i]
177-
if _, err := stmt.Exec(id, i, encode(fcr.ParentID), encode(fcr.UnlockConditions), encode(fcr.ParentID), encode(fcr.FileContract.RevisionNumber)); err != nil {
211+
for i, fcr := range fileContractRevisions {
212+
if _, err := stmt.Exec(dbID, i, encode(fcr.ParentID), encode(fcr.UnlockConditions), encode(fcr.ParentID), encode(fcr.FileContract.RevisionNumber)); err != nil {
178213
return fmt.Errorf("addFileContractRevisions: failed to execute statement: %w", err)
179214
}
180215
}
181216

182217
return nil
183218
}
184219

185-
func addStorageProofs(tx *txn, id int64, txn types.Transaction) error {
220+
func addStorageProofs(tx *txn, dbID int64, storageProofs []types.StorageProof) error {
221+
if len(storageProofs) == 0 {
222+
return nil
223+
}
224+
186225
stmt, err := tx.Prepare(`INSERT INTO transaction_storage_proofs(transaction_id, transaction_order, parent_id, leaf, proof) VALUES (?, ?, ?, ?, ?)`)
187226
if err != nil {
188227
return fmt.Errorf("addStorageProofs: failed to prepare statement: %w", err)
189228
}
190229
defer stmt.Close()
191230

192-
for i, proof := range txn.StorageProofs {
193-
if _, err := stmt.Exec(id, i, encode(proof.ParentID), proof.Leaf[:], encode(proof.Proof)); err != nil {
231+
for i, proof := range storageProofs {
232+
if _, err := stmt.Exec(dbID, i, encode(proof.ParentID), proof.Leaf[:], encode(proof.Proof)); err != nil {
194233
return fmt.Errorf("addStorageProofs: failed to execute statement: %w", err)
195234
}
196235
}
@@ -202,7 +241,7 @@ type txnDBId struct {
202241
exist bool
203242
}
204243

205-
func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[types.TransactionID]txnDBId, error) {
244+
func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[types.TransactionID]bool, error) {
206245
checkTransactionStmt, err := tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`)
207246
if err != nil {
208247
return nil, fmt.Errorf("failed to prepare check transaction statement: %v", err)
@@ -221,7 +260,7 @@ func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[
221260
}
222261
defer blockTransactionsStmt.Close()
223262

224-
txnDBIds := make(map[types.TransactionID]txnDBId)
263+
txnExist := make(map[types.TransactionID]bool)
225264
for i, txn := range txns {
226265
var exist bool
227266
var dbID int64
@@ -247,53 +286,63 @@ func addTransactions(tx *txn, bid types.BlockID, txns []types.Transaction) (map[
247286
// will be true after the above query after the first time the
248287
// transaction is encountered by this loop. So we only set the value in
249288
// the map for each transaction once.
250-
if _, ok := txnDBIds[txnID]; !ok {
251-
txnDBIds[txnID] = txnDBId{id: dbID, exist: exist}
289+
if _, ok := txnExist[txnID]; !ok {
290+
txnExist[txnID] = exist
252291
}
253292

254293
if _, err := blockTransactionsStmt.Exec(encode(bid), dbID, i); err != nil {
255294
return nil, fmt.Errorf("failed to insert into block_transactions: %w", err)
256295
}
257296
}
258297

259-
return txnDBIds, nil
298+
return txnExist, nil
260299
}
261300

262-
func addTransactionFields(tx *txn, txns []types.Transaction, txnDBIds map[types.TransactionID]txnDBId) error {
301+
func addTransactionFields(tx *txn, txns []types.Transaction, txnExist map[types.TransactionID]bool) error {
302+
stmt, err := tx.Prepare(`SELECT id FROM transactions WHERE transaction_id = ?`)
303+
if err != nil {
304+
return fmt.Errorf("failed to prepare transaction ID statement: %w", err)
305+
}
306+
263307
for _, txn := range txns {
264308
txnID := txn.ID()
265-
dbID, ok := txnDBIds[txnID]
309+
exist, ok := txnExist[txnID]
266310
if !ok {
267-
panic(fmt.Errorf("txn %v should be in txnDBIds", txn.ID()))
311+
panic(fmt.Errorf("txn %v should be in txnExist", txn.ID()))
268312
}
269313

270314
// transaction already exists, don't reinsert its fields
271-
if dbID.exist {
315+
if exist {
272316
continue
273317
}
274318
// set exist = true so we don't re-insert fields in case we have
275319
// multiple of the same transaction in a block
276-
txnDBIds[txnID] = txnDBId{id: dbID.id, exist: true}
320+
txnExist[txnID] = true
321+
322+
var dbID int64
323+
if err := stmt.QueryRow(encode(txnID)).Scan(&dbID); err != nil {
324+
return fmt.Errorf("failed to scan for transaction ID: %w", err)
325+
}
277326

278-
if err := addMinerFees(tx, dbID.id, txn); err != nil {
327+
if err := addMinerFees(tx, dbID, txn.MinerFees); err != nil {
279328
return fmt.Errorf("failed to add miner fees: %w", err)
280-
} else if err := addArbitraryData(tx, dbID.id, txn); err != nil {
329+
} else if err := addArbitraryData(tx, dbID, txn.ArbitraryData); err != nil {
281330
return fmt.Errorf("failed to add arbitrary data: %w", err)
282-
} else if err := addSignatures(tx, dbID.id, txn); err != nil {
331+
} else if err := addSignatures(tx, dbID, txn.Signatures); err != nil {
283332
return fmt.Errorf("failed to add signatures: %w", err)
284-
} else if err := addSiacoinInputs(tx, dbID.id, txn); err != nil {
333+
} else if err := addSiacoinInputs(tx, dbID, txn.SiacoinInputs); err != nil {
285334
return fmt.Errorf("failed to add siacoin inputs: %w", err)
286-
} else if err := addSiacoinOutputs(tx, dbID.id, txn); err != nil {
335+
} else if err := addSiacoinOutputs(tx, dbID, txn); err != nil {
287336
return fmt.Errorf("failed to add siacoin outputs: %w", err)
288-
} else if err := addSiafundInputs(tx, dbID.id, txn); err != nil {
337+
} else if err := addSiafundInputs(tx, dbID, txn.SiafundInputs); err != nil {
289338
return fmt.Errorf("failed to add siafund inputs: %w", err)
290-
} else if err := addSiafundOutputs(tx, dbID.id, txn); err != nil {
339+
} else if err := addSiafundOutputs(tx, dbID, txn); err != nil {
291340
return fmt.Errorf("failed to add siafund outputs: %w", err)
292-
} else if err := addFileContracts(tx, dbID.id, txn); err != nil {
341+
} else if err := addFileContracts(tx, dbID, txn); err != nil {
293342
return fmt.Errorf("failed to add file contract: %w", err)
294-
} else if err := addFileContractRevisions(tx, dbID.id, txn); err != nil {
343+
} else if err := addFileContractRevisions(tx, dbID, txn.FileContractRevisions); err != nil {
295344
return fmt.Errorf("failed to add file contract revisions: %w", err)
296-
} else if err := addStorageProofs(tx, dbID.id, txn); err != nil {
345+
} else if err := addStorageProofs(tx, dbID, txn.StorageProofs); err != nil {
297346
return fmt.Errorf("failed to add storage proofs: %w", err)
298347
}
299348
}
@@ -578,7 +627,7 @@ func addSiafundElements(tx *txn, index types.ChainIndex, spentElements, newEleme
578627
return nil
579628
}
580629

581-
func addEvents(tx *txn, bid types.BlockID, txnDBIds map[types.TransactionID]txnDBId, v2TxnDBIds map[types.TransactionID]txnDBId, events []explorer.Event) error {
630+
func addEvents(tx *txn, bid types.BlockID, events []explorer.Event) error {
582631
if len(events) == 0 {
583632
return nil
584633
}
@@ -601,13 +650,13 @@ func addEvents(tx *txn, bid types.BlockID, txnDBIds map[types.TransactionID]txnD
601650
}
602651
defer relevantAddrStmt.Close()
603652

604-
v1TransactionEventStmt, err := tx.Prepare(`INSERT INTO v1_transaction_events (event_id, transaction_id) VALUES (?, ?)`)
653+
v1TransactionEventStmt, err := tx.Prepare(`INSERT INTO v1_transaction_events (event_id, transaction_id) VALUES (?, (SELECT id FROM transactions WHERE transaction_id = ?))`)
605654
if err != nil {
606655
return fmt.Errorf("failed to prepare v1 transaction event statement: %w", err)
607656
}
608657
defer v1TransactionEventStmt.Close()
609658

610-
v2TransactionEventStmt, err := tx.Prepare(`INSERT INTO v2_transaction_events (event_id, transaction_id) VALUES (?, ?)`)
659+
v2TransactionEventStmt, err := tx.Prepare(`INSERT INTO v2_transaction_events (event_id, transaction_id) VALUES (?, (SELECT id FROM v2_transactions WHERE transaction_id = ?))`)
611660
if err != nil {
612661
return fmt.Errorf("failed to prepare v2 transaction event statement: %w", err)
613662
}
@@ -662,13 +711,11 @@ func addEvents(tx *txn, bid types.BlockID, txnDBIds map[types.TransactionID]txnD
662711

663712
switch v := event.Data.(type) {
664713
case explorer.EventV1Transaction:
665-
dbID := txnDBIds[types.TransactionID(event.ID)].id
666-
if _, err = v1TransactionEventStmt.Exec(eventID, dbID); err != nil {
714+
if _, err = v1TransactionEventStmt.Exec(eventID, encode(types.TransactionID(event.ID))); err != nil {
667715
return fmt.Errorf("failed to insert transaction event: %w", err)
668716
}
669717
case explorer.EventV2Transaction:
670-
dbID := v2TxnDBIds[types.TransactionID(event.ID)].id
671-
if _, err = v2TransactionEventStmt.Exec(eventID, dbID); err != nil {
718+
if _, err = v2TransactionEventStmt.Exec(eventID, encode(types.TransactionID(event.ID))); err != nil {
672719
return fmt.Errorf("failed to insert transaction event: %w", err)
673720
}
674721
case explorer.EventPayout:
@@ -966,12 +1013,12 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
9661013
return fmt.Errorf("ApplyIndex: failed to update matured balances: %w", err)
9671014
}
9681015

969-
txnDBIds, err := addTransactions(ut.tx, state.Block.ID(), state.Block.Transactions)
1016+
txnSeen, err := addTransactions(ut.tx, state.Block.ID(), state.Block.Transactions)
9701017
if err != nil {
9711018
return fmt.Errorf("ApplyIndex: failed to add transactions: %w", err)
9721019
}
9731020

974-
v2TxnDBIds, err := addV2Transactions(ut.tx, state.Block.ID(), state.Block.V2Transactions())
1021+
v2TxnSeen, err := addV2Transactions(ut.tx, state.Block.ID(), state.Block.V2Transactions())
9751022
if err != nil {
9761023
return fmt.Errorf("ApplyIndex: failed to add v2 transactions: %w", err)
9771024
}
@@ -994,9 +1041,9 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
9941041
return fmt.Errorf("ApplyIndex: failed to add file contracts: %w", err)
9951042
} else if err := updateV2FileContractElements(ut.tx, false, state.Metrics.Index, state.Block, state.V2FileContractElements); err != nil {
9961043
return fmt.Errorf("ApplyIndex: failed to add v2 file contracts: %w", err)
997-
} else if err := addTransactionFields(ut.tx, state.Block.Transactions, txnDBIds); err != nil {
1044+
} else if err := addTransactionFields(ut.tx, state.Block.Transactions, txnSeen); err != nil {
9981045
return fmt.Errorf("ApplyIndex: failed to add transaction fields: %w", err)
999-
} else if err := addV2TransactionFields(ut.tx, state.Block.V2Transactions(), v2TxnDBIds); err != nil {
1046+
} else if err := addV2TransactionFields(ut.tx, state.Block.V2Transactions(), v2TxnSeen); err != nil {
10001047
return fmt.Errorf("ApplyIndex: failed to add v2 transaction fields: %w", err)
10011048
} else if err := updateBalances(ut.tx, state.Metrics.Index.Height, state.SpentSiacoinElements, state.NewSiacoinElements, state.SpentSiafundElements, state.NewSiafundElements); err != nil {
10021049
return fmt.Errorf("ApplyIndex: failed to update balances: %w", err)
@@ -1012,7 +1059,7 @@ func (ut *updateTx) ApplyIndex(state explorer.UpdateState) error {
10121059
return fmt.Errorf("ApplyIndex: failed to update file contract element indices: %w", err)
10131060
} else if err := updateV2FileContractIndices(ut.tx, false, state.Metrics.Index, state.V2FileContractElements); err != nil {
10141061
return fmt.Errorf("ApplyIndex: failed to update v2 file contract element indices: %w", err)
1015-
} else if err := addEvents(ut.tx, state.Block.ID(), txnDBIds, v2TxnDBIds, state.Events); err != nil {
1062+
} else if err := addEvents(ut.tx, state.Block.ID(), state.Events); err != nil {
10161063
return fmt.Errorf("ApplyIndex: failed to add events: %w", err)
10171064
}
10181065

0 commit comments

Comments
 (0)