Merge branch 'investigate_output_cut_through'

Author: setavenger

src/common/types/utxo.go

@@ -19,7 +19,7 @@ type UTXO struct {
 	BlockHeight  uint32 `json:"block_height"` // not used
 	BlockHash    string `json:"block_hash"`
 	Timestamp    uint64 `json:"timestamp"` // not used
-	Spent        bool   `json:"-"`         // not used
+	Spent        bool   `json:"spent"`
 }
 
 // SpentUTXO
@@ -59,12 +59,28 @@ func (v *UTXO) SerialiseData() ([]byte, error) {
 		common.ErrorLogger.Println(err)
 		return nil, err
 	}
-	return buf.Bytes(), nil
+	err = binary.Write(&buf, binary.BigEndian, v.Timestamp)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return nil, err
+	}
+	err = binary.Write(&buf, binary.BigEndian, v.Spent)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return nil, err
+	}
+	data := buf.Bytes()
+	if len(data) != 34+8+8+1 {
+		common.ErrorLogger.Printf("wrong data length: %d %+v", len(data), data)
+		return nil, err
+	}
+
+	return data, nil
 }
 
 func (v *UTXO) DeSerialiseKey(key []byte) error {
 	if len(key) != 32+32+4 {
-		common.ErrorLogger.Printf("wrong key length: %+v", key)
+		common.ErrorLogger.Printf("wrong key length: %d %+v", len(key), key)
 		return errors.New("key is wrong length. should not happen")
 	}
 
@@ -79,12 +95,22 @@ func (v *UTXO) DeSerialiseKey(key []byte) error {
 }
 
 func (v *UTXO) DeSerialiseData(data []byte) error {
-	if len(data) != 34+8 {
-		common.ErrorLogger.Printf("wrong data length: %+v", data)
+	if len(data) != 34+8+8+1 {
+		common.ErrorLogger.Printf("wrong data length: %d %+v", len(data), data)
 		return errors.New("data is wrong length. should not happen")
 	}
 	v.ScriptPubKey = hex.EncodeToString(data[:34])
-	err := binary.Read(bytes.NewReader(data[34:]), binary.BigEndian, &v.Value)
+	err := binary.Read(bytes.NewReader(data[34:34+8]), binary.BigEndian, &v.Value)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+	err = binary.Read(bytes.NewReader(data[34+8:34+8+8]), binary.BigEndian, &v.Timestamp)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+	err = binary.Read(bytes.NewReader(data[34+8+8:]), binary.BigEndian, &v.Spent)
 	if err != nil {
 		common.ErrorLogger.Println(err)
 		return err

src/core/cleanup.go

@@ -85,6 +85,81 @@ func removeSpentUTXOsAndTweaks(utxos []types.UTXO) error {
 	return err
 }
 
+func markSpentUTXOsAndTweaks(utxos []types.UTXO) error {
+	if len(utxos) == 0 {
+		return nil
+	}
+	// First overwrite the spend UTXOs which now have the spent flag set
+	err := dblevel.InsertUTXOs(utxos)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	// Now begin process to find the tweaks that need to be deleted
+
+	// we only need to check for one utxo per txid, so we reduce the number of utxos -> fewer lookups in DB
+	var cleanUTXOs []types.UTXO
+	includedTxids := make(map[string]bool)
+
+	for _, utxo := range utxos {
+		if _, exists := includedTxids[utxo.Txid]; !exists {
+			cleanUTXOs = append(cleanUTXOs, utxo)
+			includedTxids[utxo.Txid] = true
+		}
+	}
+
+	var tweaksToDelete []types.Tweak
+	var tweaksToOverwrite []types.Tweak
+
+	for _, utxo := range cleanUTXOs {
+		var remainingUTXOs []types.UTXO
+		remainingUTXOs, err = dblevel.FetchByBlockHashAndTxidUTXOs(utxo.BlockHash, utxo.Txid)
+		if err != nil && !errors.Is(err, dblevel.NoEntryErr{}) {
+			common.ErrorLogger.Println(err)
+			return err
+		} else if err != nil && errors.Is(err, dblevel.NoEntryErr{}) {
+			//	if no UTXOs are left for a certain blockHash-txid combination we can remove the tweak
+			tweaksToDelete = append(tweaksToDelete, types.Tweak{
+				// we only need those Fields to serialise the key
+				BlockHash: utxo.BlockHash,
+				Txid:      utxo.Txid,
+			})
+			continue
+		}
+		var newBiggest *uint64
+		newBiggest, err = types.FindBiggestRemainingUTXO(utxo, remainingUTXOs)
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return err
+		}
+		if newBiggest != nil {
+			// find the biggest UTXO for the tx and overwrite if necessary
+			tweaksToOverwrite = append(tweaksToOverwrite, types.Tweak{
+				BlockHash:    utxo.BlockHash,
+				BlockHeight:  0,
+				Txid:         utxo.Txid,
+				Data:         [33]byte{},
+				HighestValue: *newBiggest,
+			})
+		}
+	}
+
+	err = dblevel.DeleteBatchTweaks(tweaksToDelete)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	err = dblevel.OverWriteTweaks(tweaksToOverwrite)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	return err
+}
+
 // ReindexDustLimitsOnly this routine adds the dust limit data to tweaks after a sync
 func ReindexDustLimitsOnly() error {
 	common.InfoLogger.Println("Reindexing dust limit from synced data")

src/core/extractutxos.go

@@ -71,11 +71,12 @@ func extractSpentTaprootPubKeysFromTx(tx *types.Transaction, block *types.Block)
 
 			//832974
 			spentUTXOs = append(spentUTXOs, types.UTXO{
-				Txid:      vin.Txid,
-				Vout:      vin.Vout,
-				Value:     common.ConvertFloatBTCtoSats(vin.Prevout.Value),
-				BlockHash: blockHash,
-				Spent:     true,
+				Txid:         vin.Txid,
+				Vout:         vin.Vout,
+				Value:        common.ConvertFloatBTCtoSats(vin.Prevout.Value),
+				ScriptPubKey: vin.Prevout.ScriptPubKey.Hex,
+				BlockHash:    blockHash,
+				Spent:        true,
 			})
 		default:
 			continue

src/core/routine.go

@@ -13,7 +13,7 @@ func CheckForNewBlockRoutine() {
 	common.InfoLogger.Println("starting check_for_new_block_routine")
 	for true {
 		select {
-		case <-time.NewTicker(15 * time.Second).C:
+		case <-time.NewTicker(3 * time.Second).C:
 			blockHash, err := GetBestBlockHash()
 			if err != nil {
 				common.ErrorLogger.Println(err)
@@ -170,7 +170,8 @@ func HandleBlock(block *types.Block) error {
 	// get spent taproot UTXOs
 	taprootSpent := extractSpentTaprootPubKeysFromBlock(block)
 
-	err = removeSpentUTXOsAndTweaks(taprootSpent)
+	//err = removeSpentUTXOsAndTweaks(taprootSpent)
+	err = markSpentUTXOsAndTweaks(taprootSpent)
 	if err != nil {
 		common.ErrorLogger.Println(err)
 		return err

src/db/dblevel/tweak.go

@@ -42,7 +42,7 @@ func OverWriteTweaks(tweaks []types.Tweak) error {
 			common.WarningLogger.Println("no entries for a tweak were found. this should not happen")
 			return err
 		}
-		if len(pairs) != 1 {
+		if len(pairs) != 1 { // todo understand what's going on here raises an error
 			common.ErrorLogger.Printf("%+v", pairs)
 			common.ErrorLogger.Println("number of tweaks was not exactly 1")
 			return errors.New("number of tweaks was not exactly 1")

src/server/api.go

@@ -78,7 +78,7 @@ func (h *ApiHandler) GetCFilterByHeight(c *gin.Context) {
 	c.JSON(200, data)
 }
 
-func (h *ApiHandler) GetLightUTXOsByHeight(c *gin.Context) {
+func (h *ApiHandler) GetUtxosByHeight(c *gin.Context) {
 	heightStr := c.Param("blockheight")
 	if heightStr == "" {
 		c.JSON(http.StatusBadRequest, nil)

src/server/run.go

@@ -13,7 +13,7 @@ func RunServer(api *ApiHandler) {
 	router.GET("/tweaks/:blockheight", api.GetTweakDataByHeight)
 	router.GET("/tweak-index/:blockheight", api.GetTweakIndexDataByHeight)
 	router.GET("/filter/:blockheight", api.GetCFilterByHeight)
-	router.GET("/utxos/:blockheight", api.GetLightUTXOsByHeight)
+	router.GET("/utxos/:blockheight", api.GetUtxosByHeight)
 
 	router.POST("/forward-tx", api.ForwardRawTX)