Merge pull request #9 from setavenger/spent_filter

Spent filter and Index added

Author: setavenger

README.md

@@ -9,6 +9,10 @@ matches [other implementations](https://github.com/bitcoin/bitcoin/pull/28241#is
 
 The installation process is still very manual. Will be improved based on feedback and new findings. It is advised to look at the example [blindbit.toml](blindbit.example.toml). As new config options appear they will be listed and explained there.
 
+## Breaking changes
+
+- Endpoints were expanded and have a slightly different syntax now [see endpoints](#endpoints)
+
 ### Requirements
 
 - RPC access to a bitcoin full node
@@ -84,8 +88,10 @@ No known issues.
 ```text
 GET("/block-height")  // returns the height of the indexing server
 GET("/tweaks/:blockheight?dustLimit=<sat_amount>")  // returns tweak data (cut-through); optional parameter dustLimit can be omitted; filtering happens per request, so virtually any amount can be specified
-GET("/tweak-index/:blockheight")  // returns the full tweak index (no cut-through)
-GET("/filter/:blockheight") // returns a custom taproot only filter (the underlying data is subject to change; changing scriptPubKey to x-only pubKey) 
+GET("/tweak-index/:blockheight?dustLimit=<sat_amount>")  // returns the full tweak index (no cut-through); optional parameter dustLimit can be omitted; filtering happens per request, so virtually any amount can be specified
+GET("/spent-index/:blockheight")  // returns the spent outpoints index (see https://github.com/setavenger/BIP0352-light-client-specification?tab=readme-ov-file#spent-utxos)
+GET("/filter/spent/:blockheight") // returns a filter for shortened spent outpoint hashes (see https://github.com/setavenger/BIP0352-light-client-specification?tab=readme-ov-file#filters)
+GET("/filter/new-utxos/:blockheight") // returns a custom taproot only filter of x-only pubkey which received funds
 GET("/utxos/:blockheight")  // UTXO data for that block (cut down to the essentials needed to spend)
 ```
 

src/common/config.go

@@ -79,7 +79,7 @@ func LoadConfigs(pathToConfig string) {
 		WarningLogger.Println("make sure your configuration loaded correctly, check example blindbit.toml for configuration")
 	}
 
-	if TweaksCutThroughWithDust && !TweaksOnly {
+	if TweaksCutThroughWithDust && TweaksOnly {
 		err := errors.New("cut through requires tweaks_only to be set to 1")
 		ErrorLogger.Println(err)
 		os.Exit(1)

src/common/types/spentoutpoints.go

@@ -0,0 +1,77 @@
+package types
+
+import (
+	"bytes"
+	"encoding/hex"
+	"errors"
+
+	"SilentPaymentAppBackend/src/common"
+)
+
+const LenOutpointHashShort = 8
+
+type SpentOutpointsIndex struct {
+	BlockHash   string                       `json:"block_hash"`
+	BlockHeight uint32                       `json:"block_height"`
+	Data        [][LenOutpointHashShort]byte `json:"data"`
+}
+
+func PairFactorySpentOutpointsIndex() Pair {
+	var filter Pair = &SpentOutpointsIndex{}
+	return filter
+}
+
+func (v *SpentOutpointsIndex) SerialiseKey() ([]byte, error) {
+	return GetDBKeyTweakIndex(v.BlockHash)
+}
+
+func (v *SpentOutpointsIndex) SerialiseData() ([]byte, error) {
+
+	// todo can this be made more efficiently?
+	totalLength := len(v.Data) * LenOutpointHashShort
+	flattened := make([]byte, 0, totalLength)
+
+	for _, byteArray := range v.Data {
+		flattened = append(flattened, byteArray[:]...)
+	}
+
+	return flattened, nil
+}
+
+func (v *SpentOutpointsIndex) DeSerialiseKey(key []byte) error {
+	if len(key) != 32 {
+		common.ErrorLogger.Printf("wrong key length: %+v", key)
+		return errors.New("key is wrong length. should not happen")
+	}
+
+	v.BlockHash = hex.EncodeToString(key)
+
+	return nil
+}
+
+func (v *SpentOutpointsIndex) DeSerialiseData(data []byte) error {
+	if len(data)%LenOutpointHashShort != 0 {
+		common.ErrorLogger.Printf("wrong data length: %+v", data)
+		return errors.New("data is wrong length. should not happen")
+	}
+
+	numArrays := len(data) / LenOutpointHashShort
+	v.Data = make([][LenOutpointHashShort]byte, numArrays)
+	// Iterate and copy segments from the flat slice into the new array of arrays
+	for i := 0; i < numArrays; i++ {
+		copy(v.Data[i][:], data[i*LenOutpointHashShort:(i+1)*LenOutpointHashShort])
+	}
+	return nil
+}
+
+func GetDBKeySpentSpentOutpointsIndex(blockHash string) ([]byte, error) {
+	var buf bytes.Buffer
+	blockHashBytes, err := hex.DecodeString(blockHash)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return nil, err
+	}
+	buf.Write(blockHashBytes)
+
+	return buf.Bytes(), nil
+}

src/common/util.go

@@ -2,12 +2,14 @@ package common
 
 import (
 	"crypto/sha256"
-	"github.com/shopspring/decimal"
-	"golang.org/x/crypto/ripemd160"
 	"os/user"
 	"strings"
+
+	"github.com/shopspring/decimal"
+	"golang.org/x/crypto/ripemd160"
 )
 
+// ReverseBytes reverses the bytes inside the byte slice and returns the same slice. It does not return a copy.
 func ReverseBytes(bytes []byte) []byte {
 	for i, j := 0, len(bytes)-1; i < j; i, j = i+1, j-1 {
 		bytes[i], bytes[j] = bytes[j], bytes[i]

src/common/vars.go

@@ -58,13 +58,15 @@ var (
 
 // one has to call SetDirectories otherwise common.DBPath will be empty
 var (
-	DBPathHeaders        string
-	DBPathHeadersInv     string
-	DBPathFilters        string
-	DBPathTweaks         string
-	DBPathTweakIndex     string
-	DBPathTweakIndexDust string
-	DBPathUTXOs          string
+	DBPathHeaders              string
+	DBPathHeadersInv           string // for height to blockHash mapping
+	DBPathFilters              string
+	DBPathTweaks               string
+	DBPathTweakIndex           string
+	DBPathUTXOs                string
+	DBPathTweakIndexDust       string
+	DBPathSpentOutpointsIndex  string
+	DBPathSpentOutpointsFilter string
 )
 
 // NumsH = 0x50929b74c1a04954b78b4b6035e97a5e078a5a0f28ec96d547bfee9ace803ac0
@@ -83,6 +85,8 @@ func SetDirectories() {
 	DBPathTweakIndex = DBPath + "/tweak-index"
 	DBPathTweakIndexDust = DBPath + "/tweak-index-dust"
 	DBPathUTXOs = DBPath + "/utxos"
+	DBPathSpentOutpointsIndex = DBPath + "/spent-index"
+	DBPathSpentOutpointsFilter = DBPath + "/spent-filter"
 }
 
 func HeaderMustSyncHeight() uint32 {

src/core/cfilter.go

@@ -3,14 +3,17 @@ package core
 import (
 	"SilentPaymentAppBackend/src/common"
 	"SilentPaymentAppBackend/src/common/types"
+	"bytes"
+	"encoding/binary"
 	"encoding/hex"
+
 	"github.com/btcsuite/btcd/btcutil/gcs/builder"
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcutil/gcs"
 )
 
 // BuildTaprootOnlyFilter creates the taproot only filter
-func BuildTaprootOnlyFilter(block *types.Block) (types.Filter, error) {
+func BuildNewUTXOsFilter(block *types.Block) (types.Filter, error) {
 	var taprootOutput [][]byte
 
 	for _, tx := range block.Txs {
@@ -64,3 +67,68 @@ func BuildTaprootOnlyFilter(block *types.Block) (types.Filter, error) {
 		BlockHash:   block.Hash,
 	}, nil
 }
+
+// BuildSpentUTXOsFilter creates a filter based on the spent
+func BuildSpentUTXOsFilter(spentOutpointsIndex types.SpentOutpointsIndex) (types.Filter, error) {
+
+	blockHashBytes, err := hex.DecodeString(spentOutpointsIndex.BlockHash)
+	if err != nil {
+		common.DebugLogger.Println("blockHash", spentOutpointsIndex.BlockHash)
+		common.ErrorLogger.Fatalln(err)
+		return types.Filter{}, err
+	}
+	c := chainhash.Hash{}
+
+	err = c.SetBytes(common.ReverseBytes(blockHashBytes))
+	if err != nil {
+		common.DebugLogger.Println("blockHash", spentOutpointsIndex.BlockHash)
+		common.ErrorLogger.Fatalln(err)
+		return types.Filter{}, err
+
+	}
+	key := builder.DeriveKey(&c)
+
+	// convert to slices
+	data := make([][]byte, len(spentOutpointsIndex.Data))
+	for i, outpointHash := range spentOutpointsIndex.Data {
+		var newHash [8]byte
+		copy(newHash[:], outpointHash[:])
+		data[i] = newHash[:]
+	}
+
+	filter, err := gcs.BuildGCSFilter(builder.DefaultP, builder.DefaultM, key, data)
+	if err != nil {
+		common.ErrorLogger.Fatalln(err)
+		return types.Filter{}, err
+	}
+
+	nBytes, err := filter.NBytes()
+	if err != nil {
+		common.ErrorLogger.Fatalln(err)
+		return types.Filter{}, err
+	}
+
+	return types.Filter{
+		FilterType:  4,
+		BlockHeight: spentOutpointsIndex.BlockHeight,
+		Data:        nBytes,
+		BlockHash:   spentOutpointsIndex.BlockHash,
+	}, nil
+}
+
+func SerialiseToOutpoint(utxo types.UTXO) ([]byte, error) {
+	var buf bytes.Buffer
+
+	txidBytes, err := hex.DecodeString(utxo.Txid)
+	if err != nil {
+		common.DebugLogger.Println(utxo.Txid)
+		common.ErrorLogger.Println(err)
+		return nil, err
+	}
+
+	// err is always nil
+	buf.Write(common.ReverseBytes(txidBytes))
+
+	binary.Write(&buf, binary.LittleEndian, utxo.Value)
+	return buf.Bytes(), err
+}

src/core/extractutxos.go

@@ -75,7 +75,6 @@ func extractSpentTaprootPubKeysFromTx(tx *types.Transaction, block *types.Block)
 				blockHash = headerInv.Hash
 			}
 
-			//832974
 			spentUTXOs = append(spentUTXOs, types.UTXO{
 				Txid:         vin.Txid,
 				Vout:         vin.Vout,

src/core/routine.go

@@ -12,19 +12,17 @@ import (
 func CheckForNewBlockRoutine() {
 	common.InfoLogger.Println("starting check_for_new_block_routine")
 	for {
-		select {
-		case <-time.NewTicker(3 * time.Second).C:
-			blockHash, err := GetBestBlockHash()
-			if err != nil {
-				common.ErrorLogger.Println(err)
-				// todo fail or restart after too many fails?
-				continue
-			}
-			err = FullProcessBlockHash(blockHash)
-			if err != nil {
-				common.ErrorLogger.Println(err)
-				return
-			}
+		<-time.NewTicker(3 * time.Second).C
+		blockHash, err := GetBestBlockHash()
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			// todo fail or restart after too many fails?
+			continue
+		}
+		err = FullProcessBlockHash(blockHash)
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return
 		}
 	}
 }
@@ -58,7 +56,7 @@ func FullProcessBlockHash(blockHash string) error {
 func PullBlock(blockHash string) (*types.Block, error) {
 	if len(blockHash) != 64 {
 		common.ErrorLogger.Println("block_hash invalid:", blockHash)
-		return nil, errors.New(fmt.Sprintf("block_hash invalid: %s", blockHash))
+		return nil, fmt.Errorf("block_hash invalid: %s", blockHash)
 	}
 	// this method is preferred over lastHeader because then this function can be called for PreviousBlockHash
 	header, err := dblevel.FetchByBlockHashBlockHeader(blockHash)
@@ -213,12 +211,38 @@ func HandleBlock(block *types.Block) error {
 	}
 
 	// create special block filter
-	cFilterTaproot, err := BuildTaprootOnlyFilter(block)
+	cFilterNewUTXOs, err := BuildNewUTXOsFilter(block)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	//
+	err = dblevel.InsertNewUTXOsFilter(cFilterNewUTXOs)
 	if err != nil {
 		common.ErrorLogger.Println(err)
 		return err
 	}
-	err = dblevel.InsertFilter(cFilterTaproot)
+
+	spentOutpointsIndex, err := BuildSpentUTXOIndex(taprootSpent, block)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	err = dblevel.InsertSpentOutpointsIndex(&spentOutpointsIndex)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	cFilterSpentUTXOs, err := BuildSpentUTXOsFilter(spentOutpointsIndex)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
+	}
+
+	err = dblevel.InsertSpentOutpointsFilter(cFilterSpentUTXOs)
 	if err != nil {
 		common.ErrorLogger.Println(err)
 		return err

src/core/spentutxos.go

@@ -0,0 +1,36 @@
+package core
+
+import (
+	"SilentPaymentAppBackend/src/common"
+	"SilentPaymentAppBackend/src/common/types"
+	"crypto/sha256"
+	"encoding/hex"
+)
+
+func BuildSpentUTXOIndex(utxos []types.UTXO, block *types.Block) (types.SpentOutpointsIndex, error) {
+
+	blockHashBytes, err := hex.DecodeString(block.Hash)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return types.SpentOutpointsIndex{}, err
+	}
+
+	spentOutpointsIndex := types.SpentOutpointsIndex{
+		BlockHash:   block.Hash,
+		BlockHeight: block.Height,
+	}
+
+	for _, utxo := range utxos {
+		var outpoint []byte
+		outpoint, err = SerialiseToOutpoint(utxo)
+		if err != nil {
+			common.ErrorLogger.Println(err)
+			return types.SpentOutpointsIndex{}, err
+		}
+
+		hashedOutpoint := sha256.Sum256(append(outpoint, blockHashBytes...))
+		spentOutpointsIndex.Data = append(spentOutpointsIndex.Data, [types.LenOutpointHashShort]byte(hashedOutpoint[:]))
+	}
+
+	return spentOutpointsIndex, err
+}

src/dataexport/exportcsv.go

@@ -78,7 +78,7 @@ func convertUTXOsToRecords(utxos []types.UTXO) ([][]string, error) {
 // Filters
 
 func ExportFilters(path string) error {
-	allEntries, err := dblevel.FetchAllFilters()
+	allEntries, err := dblevel.FetchAllNewUTXOsFilters()
 	if err != nil {
 		common.ErrorLogger.Println(err)
 		return err