utxos for fully spent txs are pruned from db now

Author: setavenger

src/core/cleanup.go

@@ -11,20 +11,38 @@ func overwriteUTXOsWithLookUp(utxos []types.UTXO) error {
 	var utxosToOverwrite []types.UTXO
 
 	for _, utxo := range utxos {
-		fetchedUTXOs, err := dblevel.FetchByBlockHashAndTxidUTXOs(utxo.BlockHash, utxo.Txid)
+		_, err := dblevel.FetchByBlockHashAndTxidUTXOs(utxo.BlockHash, utxo.Txid)
 		if err != nil && !errors.Is(err, dblevel.NoEntryErr{}) {
 			common.ErrorLogger.Println(err)
 			return err
 		} else if err != nil && errors.Is(err, dblevel.NoEntryErr{}) {
 			// we skip if no entry was found. We don't want to insert those
 			continue
 		}
-		// we actually don't have to check the fetched UTXOs. if any utxos were found for this transaction it means that it was eligible.
+		// we actually don't have to check the fetched UTXOs. If any utxos were found for this transaction it means that it was eligible.
 		// hence all taproot utxos have to be present
-		_ = fetchedUTXOs
 		utxosToOverwrite = append(utxosToOverwrite, utxo)
 	}
 	err := dblevel.InsertUTXOs(utxosToOverwrite)
+	alreadyCheckedTxids := make(map[string]struct{})
+	for _, utxo := range utxosToOverwrite {
+		if _, ok := alreadyCheckedTxids[utxo.Txid]; ok {
+			continue
+		}
+		var key []byte
+		key, err = utxo.SerialiseKey()
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return err
+		}
+		_ = key
+		err = dblevel.PruneUTXOs(key[:64])
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return err
+		}
+		alreadyCheckedTxids[utxo.Txid] = struct{}{}
+	}
 	if err != nil {
 		common.ErrorLogger.Println(err)
 		return err
@@ -88,7 +106,7 @@ func markSpentUTXOsAndTweaks(utxos []types.UTXO) error {
 			return err
 		} else if err != nil && errors.Is(err, dblevel.NoEntryErr{}) {
 			// this case should not even occur at this stage as utxos are not deleted before this query and are only marked as spent
-			common.DebugLogger.Printf("txid: %x\n", utxo.Txid)
+			common.DebugLogger.Printf("txid: %s\n", utxo.Txid)
 			common.DebugLogger.Println("no UTXOs were found for transaction")
 			continue
 		}
@@ -151,12 +169,11 @@ func ReindexDustLimitsOnly() error {
 	}
 	common.InfoLogger.Println("Reindexing dust limit done")
 	return nil
-
 }
 
-// Prune
-// This function checks the utxo set and deletes all utxos of a transaction where all utxos are marked as spent.
-// it can then also remove the tweak of completely spent transaction
-func Prune() error {
-	return nil
+// PruneUTXOs
+// This function searches the UTXO set for transactions where all UTXOs are marked as spent, and removes those UTXOs.
+func PruneAllUTXOs() error {
+	common.InfoLogger.Println("Pruning All UTXOs")
+	return dblevel.PruneUTXOs(nil)
 }

src/core/routine.go

@@ -84,10 +84,12 @@ func PullBlock(blockHash string) (*types.Block, error) {
 
 // CheckBlock checks whether the block hash has already been processed and will process the block if needed
 func CheckBlock(block *types.Block) {
+	// todo add return type error
 	// todo this should fail at the highest instance were its wrapped in,
 	//  fatal made sense here while it only had one use,
 	//  but might not want to exit the program if used in other locations
 	//common.InfoLogger.Println("Processing block:", block.Height)
+	common.DebugLogger.Println("block:", block.Height)
 
 	err := HandleBlock(block)
 	if err != nil {

src/dataexport/exportcsv.go

@@ -6,13 +6,15 @@ import (
 	"SilentPaymentAppBackend/src/db/dblevel"
 	"encoding/csv"
 	"encoding/hex"
+	"fmt"
 	"log"
 	"os"
 	"strconv"
 )
 
 func writeToCSV(path string, records [][]string) error {
 	// Create a new file
+	os.MkdirAll(fmt.Sprintf("%s/export", common.BaseDirectory), 0750)
 	file, err := os.Create(path)
 	if err != nil {
 		log.Fatalf("failed creating file: %s", err)

src/db/dblevel/blockheaderinv.go

@@ -5,6 +5,7 @@ import (
 	"SilentPaymentAppBackend/src/common/types"
 	"bytes"
 	"encoding/binary"
+
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -59,7 +60,6 @@ func FetchHighestBlockHeaderInv() (*types.BlockHeaderInv, error) {
 	var result types.BlockHeaderInv
 
 	if iter.Last() {
-		// Deserialize data first
 		err := result.DeSerialiseData(iter.Value())
 		if err != nil {
 			common.ErrorLogger.Println(err)

src/db/dblevel/tweak.go

@@ -12,7 +12,7 @@ import (
 
 // InsertBatchTweaks index implements cut through and dust
 func InsertBatchTweaks(tweaks []types.Tweak) error {
-	common.DebugLogger.Println("Inserting tweaks...")
+	// common.DebugLogger.Println("Inserting tweaks...")
 	// Create a slice of types.Pair with the same length as pairs
 	pairs := make([]types.Pair, len(tweaks))
 
@@ -45,7 +45,7 @@ func OverWriteTweaks(tweaks []types.Tweak) error {
 			return err // keep this as an error. if this happens we have to know
 		}
 
-		// this will be removed as we still check
+		// this will be removed as we still test, see below
 		if len(pairs) != 1 {
 			// this scenario should never happen. The database should not have >1 entries for one transaction. <1 (0) should give no entry error
 			// prev

src/db/dblevel/utxo.go

@@ -4,10 +4,12 @@ import (
 	"SilentPaymentAppBackend/src/common"
 	"SilentPaymentAppBackend/src/common/types"
 	"errors"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 func InsertUTXOs(utxos []types.UTXO) error {
-	common.DebugLogger.Println("Inserting UTXOs...")
+	// common.DebugLogger.Println("Inserting UTXOs...")
 	// Create a slice of types.Pair with the same length as pairs
 	pairs := make([]types.Pair, len(utxos))
 
@@ -83,7 +85,7 @@ func FetchByBlockHashAndTxidUTXOs(blockHash, txid string) ([]types.UTXO, error)
 }
 
 func DeleteBatchUTXOs(utxos []types.UTXO) error {
-	common.DebugLogger.Println("Deleting UTXOs...")
+	// common.DebugLogger.Println("Deleting UTXOs...")
 	// Create a slice of types.Pair with the same length as pairs
 	pairs := make([]types.Pair, len(utxos))
 
@@ -125,3 +127,83 @@ func FetchAllUTXOs() ([]types.UTXO, error) {
 	}
 	return result, err
 }
+
+// PruneUTXOs iterates over a set of utxos according to a prefix (all if set to nil).
+// The function checks whether the utxos are eligible for removal.
+func PruneUTXOs(prefix []byte) error {
+	iter := UTXOsDB.NewIterator(util.BytesPrefix(prefix), nil)
+	defer iter.Release()
+
+	// totalSet is for the final batch deletion
+	var totalSetToDelete []types.UTXO
+
+	var lastTxid string
+	var canBeRemoved = true
+	var currentSet []types.UTXO
+
+	var err error
+
+	for iter.Next() {
+
+		var value types.UTXO
+
+		err = value.DeSerialiseKey(iter.Key())
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return err
+		}
+
+		if lastTxid == "" {
+			lastTxid = value.Txid
+		}
+
+		if !canBeRemoved && value.Txid == lastTxid {
+			continue
+		}
+
+		err = value.DeSerialiseData(iter.Value())
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return err
+		}
+
+		if value.Spent {
+			currentSet = append(currentSet, value)
+		} else {
+			canBeRemoved = false
+		}
+
+		if value.Txid != lastTxid {
+			// delete the current set of UTXOs if eligible
+
+			// do deletion
+			if lastTxid != "" && canBeRemoved {
+				totalSetToDelete = append(totalSetToDelete, currentSet...)
+				// common.DebugLogger.Printf("Added %d UTXOs for deletion - %s\n", len(currentSet), lastTxid)
+			}
+
+			// reset state
+			currentSet = nil
+			canBeRemoved = true
+		}
+		lastTxid = value.Txid
+	}
+
+	// Handle the last batch of UTXOs after the loop
+	if canBeRemoved && len(currentSet) > 0 {
+		totalSetToDelete = append(totalSetToDelete, currentSet...)
+		// common.DebugLogger.Printf("Added %d UTXOs for deletion - %s\n", len(currentSet), lastTxid)
+	}
+	err = iter.Error()
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	err = DeleteBatchUTXOs(totalSetToDelete)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+	return nil
+}

src/main.go

@@ -12,7 +12,6 @@ import (
 	"log"
 	"path"
 
-	//_ "net/http/pprof" // Import for side effects: registers pprof handlers with the default mux.
 	"os"
 	"os/signal"
 	"strings"
@@ -21,17 +20,16 @@ import (
 
 var (
 	displayVersion bool
+	pruneOnStart   bool
+	exportData     bool
 	Version        = "0.0.0"
 )
 
 func init() {
-	// for profiling or testing iirc
-	//go func() {
-	//	log.Println(http.ListenAndServe("localhost:6060", nil))
-	//}()
-
 	flag.StringVar(&common.BaseDirectory, "datadir", common.DefaultBaseDirectory, "Set the base directory for blindbit oracle. Default directory is ~/.blindbit-oracle")
 	flag.BoolVar(&displayVersion, "version", false, "show version of blindbit-oracle")
+	flag.BoolVar(&pruneOnStart, "reprune", false, "set this flag if you want to prune on startup")
+	flag.BoolVar(&exportData, "export-data", false, "export the databases")
 	flag.Parse()
 
 	if displayVersion {
@@ -123,10 +121,20 @@ func main() {
 	// make sure everything is ready before we receive data
 
 	//todo create proper handling for exporting data
-	//exportAll()
+
+	if exportData {
+		common.InfoLogger.Println("Exporting data")
+		dataexport.ExportUTXOs(fmt.Sprintf("%s/export/utxos.csv", common.BaseDirectory))
+		return
+	}
 
 	//moved into go routine such that the interrupt signal will apply properly
 	go func() {
+		if pruneOnStart {
+			startPrune := time.Now()
+			core.PruneAllUTXOs()
+			common.InfoLogger.Printf("Pruning took: %s", time.Since(startPrune).String())
+		}
 		startSync := time.Now()
 		err := core.PreSyncHeaders()
 		if err != nil {

src/server/middleware.go

@@ -28,7 +28,7 @@ func FetchHeaderInvMiddleware(c *gin.Context) {
 	headerInv, err := dblevel.FetchByBlockHeightBlockHeaderInv(uint32(height))
 	if err != nil {
 		common.ErrorLogger.Println(err)
-		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not fetch header inventory"})
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "could not fetch header inv"})
 		c.Abort()
 		return
 	}