dcrpg:remove duplicates for votes, misses and treasury table

Signed-off-by: Philemon Ukane <[email protected]>

Author: ukane-philemon

db/dcrpg/indexing.go

@@ -394,6 +394,18 @@ func (pgb *ChainDB) deleteDuplicateAgendaVotes() (int64, error) {
 	return deleteDuplicateAgendaVotes(pgb.db)
 }
 
+func (pgb *ChainDB) DeleteDuplicateVotes() (int64, error) {
+	return DeleteDuplicateVotes(pgb.db)
+}
+
+func (pgb *ChainDB) DeleteDuplicateMisses() (int64, error) {
+	return DeleteDuplicateMisses(pgb.db)
+}
+
+func (pgb *ChainDB) DeleteDuplicateTreasuryTxs() (int64, error) {
+	return DeleteDuplicateTreasuryTxs(pgb.db)
+}
+
 // Indexes checks
 
 // MissingIndexes lists missing table indexes and their descriptions.

db/dcrpg/internal/stakestmts.go

@@ -414,6 +414,35 @@ const (
 				FROM agenda_votes) t
 			WHERE t.rnum > 1);`
 
+	// DeleteVotesDuplicateRows removes rows that would violate the unique
+	// index uix_votes_hashes_index. This should be run prior to creating the index.
+	DeleteVotesDuplicateRows = `DELETE FROM votes
+	WHERE id IN (SELECT id FROM (
+			SELECT id,
+				row_number() OVER (PARTITION BY tx_hash, block_hash ORDER BY id DESC) AS rnum
+			FROM votes) t
+		WHERE t.rnum > 1);`
+
+	// DeleteMissesDuplicateRows removes rows that would violate the unique
+	// index uix_misses_hashes_index. This should be run prior to creating the index.
+	DeleteMissesDuplicateRows = `DELETE FROM misses
+	WHERE id IN (SELECT id FROM (
+			SELECT id,
+				row_number() OVER (PARTITION BY ticket_hash, block_hash ORDER BY id DESC) AS rnum
+			FROM misses) t
+		WHERE t.rnum > 1);`
+
+	// DeleteTreasuryTxsDuplicateRows removes rows that would violate the unique
+	// index uix_treasury_tx_hash. This should be run prior to creating the index.
+	DeleteTreasuryTxsDuplicateRows = `DELETE FROM treasury
+	WHERE (tx_hash, block_hash) IN (SELECT tx_hash, block_hash FROM (
+			SELECT 
+			tx_hash,
+			block_hash,
+				row_number() OVER (PARTITION BY tx_hash, block_hash) AS rnum
+			FROM treasury) t
+		WHERE t.rnum > 1);`
+
 	// Select
 
 	SelectAgendasVotesByTime = `SELECT votes.block_time AS timestamp,` +

db/dcrpg/queries.go

@@ -257,6 +257,42 @@ func deleteDuplicateAgendaVotes(db *sql.DB) (int64, error) {
 	return sqlExec(db, internal.DeleteAgendaVotesDuplicateRows, execErrPrefix)
 }
 
+// DeleteDuplicateVotes deletes rows in votes with duplicate tx_hash and
+// block_hash leaving one row with the lowest id.
+func DeleteDuplicateVotes(db *sql.DB) (int64, error) {
+	if isuniq, err := IsUniqueIndex(db, internal.IndexOfVotesTableOnHashes); err != nil && err != sql.ErrNoRows {
+		return 0, err
+	} else if isuniq {
+		return 0, nil
+	}
+	execErrPrefix := "failed to delete duplicate votes: "
+	return sqlExec(db, internal.DeleteVotesDuplicateRows, execErrPrefix)
+}
+
+// DeleteDuplicateMisses deletes rows in misses with duplicate ticket_hash and
+// block_hash leaving one row with the lowest id.
+func DeleteDuplicateMisses(db *sql.DB) (int64, error) {
+	if isuniq, err := IsUniqueIndex(db, internal.IndexOfMissesTableOnHashes); err != nil && err != sql.ErrNoRows {
+		return 0, err
+	} else if isuniq {
+		return 0, nil
+	}
+	execErrPrefix := "failed to delete duplicate misses: "
+	return sqlExec(db, internal.DeleteMissesDuplicateRows, execErrPrefix)
+}
+
+// DeleteDuplicateTreasuryTxs deletes rows in misses with duplicate tx_hash and
+// block_hash leaving one row with the lowest id.
+func DeleteDuplicateTreasuryTxs(db *sql.DB) (int64, error) {
+	if isuniq, err := IsUniqueIndex(db, internal.IndexOfTreasuryTableOnTxHash); err != nil && err != sql.ErrNoRows {
+		return 0, err
+	} else if isuniq {
+		return 0, nil
+	}
+	execErrPrefix := "failed to delete duplicate treasury txs: "
+	return sqlExec(db, internal.DeleteTreasuryTxsDuplicateRows, execErrPrefix)
+}
+
 // --- stake (votes, tickets, misses, treasury) tables ---
 
 func insertTreasuryTxns(db *sql.DB, dbTxns []*dbtypes.Tx, checked, updateExistingRecords bool) error {

db/dcrpg/tables.go

@@ -218,6 +218,15 @@ func (pgb *ChainDB) deleteDuplicates(barLoad chan *dbtypes.ProgressBarLoad) erro
 
 		// Remove duplicate agenda_votes
 		{TableName: "agenda_votes", DropDupsFunc: pgb.deleteDuplicateAgendaVotes},
+
+		// Remove duplicate votes
+		{TableName: "votes", DropDupsFunc: pgb.DeleteDuplicateVotes},
+
+		// Remove duplicate misses
+		{TableName: "misses", DropDupsFunc: pgb.DeleteDuplicateMisses},
+
+		// Remove duplicate treasury txs
+		{TableName: "treasury", DropDupsFunc: pgb.DeleteDuplicateTreasuryTxs},
 	}
 
 	var err error