Hex strings to byte slices/arrays

Redesigned to have byte slices/arrays instead of strings for almost
all relevant fields

Author: setavenger

go.mod

@@ -1,37 +1,39 @@
 module github.com/setavenger/blindbit-oracle
 
-go 1.20
+go 1.24.1
 
 require (
-	github.com/btcsuite/btcd/btcec/v2 v2.3.3
-	github.com/btcsuite/btcd/btcutil v1.1.5
+	github.com/btcsuite/btcd/btcutil v1.1.6
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0
 	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/gin-contrib/cors v1.7.2
-	github.com/gin-gonic/gin v1.9.1
-	github.com/shopspring/decimal v1.3.1
-	github.com/spf13/viper v1.18.2
+	github.com/gin-contrib/gzip v1.2.2
+	github.com/gin-gonic/gin v1.10.0
+	github.com/rs/zerolog v1.34.0
+	github.com/setavenger/blindbit-lib v0.0.0-20250807130019-f9642edb8c97
+	github.com/setavenger/go-bip352 v0.1.8-0.20250807125845-136879952399
+	github.com/setavenger/go-libsecp256k1 v0.0.0
+	github.com/spf13/viper v1.19.0
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
-	golang.org/x/crypto v0.22.0
 )
 
 require (
 	github.com/aead/siphash v1.0.1 // indirect
-	github.com/btcsuite/btcd v0.23.5-0.20231215221805-96c9fd8078fd // indirect
+	github.com/btcsuite/btcd v0.24.2 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.3.5 // indirect
 	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
 	github.com/bytedance/sonic v1.11.6 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
-	github.com/cloudwego/iasm v0.2.0 // indirect
-	github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.24.0 // indirect
-	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 // indirect
 	github.com/goccy/go-json v0.10.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -44,16 +46,19 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/setavenger/go-bip352 v0.1.6 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect

internal/core/block_test.go

@@ -23,7 +23,7 @@ func TestBlockAnalysis(t *testing.T) {
 	}
 
 	for _, tweak := range tweaks {
-		logging.L.Info().Hex("tweak", tweak.TweakData[:]).Str("txid", tweak.Txid).Msg("tweak")
+		logging.L.Info().Hex("tweak", tweak.TweakData[:]).Hex("txid", tweak.Txid[:]).Msg("tweak")
 	}
 
 	for _, tx := range block.Txs {

internal/core/cfilter.go

@@ -1,7 +1,6 @@
 package core
 
 import (
-	"bytes"
 	"encoding/binary"
 	"encoding/hex"
 
@@ -61,26 +60,27 @@ func BuildNewUTXOsFilter(block *types.Block) (types.Filter, error) {
 		return types.Filter{}, err
 	}
 
+	blockhashBytes, err := hex.DecodeString(block.Hash)
+	if err != nil {
+		logging.L.Fatal().Err(err).Str("blockhash", block.Hash).Msg("Failed to decode block hash")
+		return types.Filter{}, err
+	}
+
 	return types.Filter{
 		FilterType:  4,
 		BlockHeight: block.Height,
 		Data:        nBytes,
-		BlockHash:   block.Hash,
+		BlockHash:   [32]byte(blockhashBytes),
 	}, nil
 }
 
 // BuildSpentUTXOsFilter creates a filter based on the spent
 func BuildSpentUTXOsFilter(spentOutpointsIndex types.SpentOutpointsIndex) (types.Filter, error) {
-	blockHashBytes, err := hex.DecodeString(spentOutpointsIndex.BlockHash)
-	if err != nil {
-		logging.L.Fatal().Err(err).Str("blockhash", spentOutpointsIndex.BlockHash).Msg("Failed to decode block hash")
-		return types.Filter{}, err
-	}
 	c := chainhash.Hash{}
 
-	err = c.SetBytes(utils.ReverseBytes(blockHashBytes))
+	err := c.SetBytes(utils.ReverseBytes(spentOutpointsIndex.BlockHash[:]))
 	if err != nil {
-		logging.L.Fatal().Err(err).Str("blockhash", spentOutpointsIndex.BlockHash).Msg("Failed to set block hash")
+		logging.L.Fatal().Err(err).Hex("blockhash", spentOutpointsIndex.BlockHash[:]).Msg("Failed to set block hash")
 		return types.Filter{}, err
 
 	}
@@ -96,13 +96,13 @@ func BuildSpentUTXOsFilter(spentOutpointsIndex types.SpentOutpointsIndex) (types
 
 	filter, err := gcs.BuildGCSFilter(builder.DefaultP, builder.DefaultM, key, data)
 	if err != nil {
-		logging.L.Fatal().Err(err).Str("blockhash", spentOutpointsIndex.BlockHash).Msg("Failed to build GCS filter")
+		logging.L.Fatal().Err(err).Hex("blockhash", spentOutpointsIndex.BlockHash[:]).Msg("Failed to build GCS filter")
 		return types.Filter{}, err
 	}
 
 	nBytes, err := filter.NBytes()
 	if err != nil {
-		logging.L.Fatal().Err(err).Str("blockhash", spentOutpointsIndex.BlockHash).Msg("Failed to get NBytes")
+		logging.L.Fatal().Err(err).Hex("blockhash", spentOutpointsIndex.BlockHash[:]).Msg("Failed to get NBytes")
 		return types.Filter{}, err
 	}
 
@@ -115,17 +115,10 @@ func BuildSpentUTXOsFilter(spentOutpointsIndex types.SpentOutpointsIndex) (types
 }
 
 func SerialiseToOutpoint(utxo types.UTXO) ([]byte, error) {
-	var buf bytes.Buffer
-
-	txidBytes, err := hex.DecodeString(utxo.Txid)
-	if err != nil {
-		logging.L.Fatal().Err(err).Str("txid", utxo.Txid).Msg("Failed to decode txid")
-		return nil, err
-	}
+	out := make([]byte, 32+4)
 
-	// err is always nil
-	buf.Write(utils.ReverseBytes(txidBytes))
+	copy(out[:32], utils.ReverseBytesCopy(utxo.Txid[:]))
+	binary.LittleEndian.PutUint32(out[32:], utxo.Vout)
 
-	binary.Write(&buf, binary.LittleEndian, utxo.Vout)
-	return buf.Bytes(), err
+	return out, nil
 }

internal/core/cleanup.go

@@ -28,7 +28,7 @@ func overwriteUTXOsWithLookUp(utxos []types.UTXO) error {
 		utxosToOverwrite = append(utxosToOverwrite, &utxo)
 	}
 	err := dblevel.InsertUTXOs(utxosToOverwrite)
-	alreadyCheckedTxids := make(map[string]struct{})
+	alreadyCheckedTxids := make(map[[32]byte]struct{})
 	for _, utxo := range utxosToOverwrite {
 		if _, ok := alreadyCheckedTxids[utxo.Txid]; ok {
 			continue
@@ -90,7 +90,7 @@ func markSpentUTXOsAndTweaks(utxos []types.UTXO) error {
 
 	// we only need to check for one utxo per txid, so we reduce the number of utxos -> fewer lookups in DB
 	var cleanUTXOs []types.UTXO
-	includedTxids := make(map[string]bool)
+	includedTxids := make(map[[32]byte]bool)
 
 	for _, utxo := range utxos {
 		if _, exists := includedTxids[utxo.Txid]; !exists {

internal/core/extractutxos.go

@@ -1,6 +1,8 @@
 package core
 
 import (
+	"encoding/hex"
+
 	"github.com/setavenger/blindbit-lib/logging"
 	"github.com/setavenger/blindbit-lib/utils"
 
@@ -19,14 +21,18 @@ func ExtractNewUTXOs(block *types.Block, eligible map[string]struct{}) []*types.
 		}
 		for _, vout := range tx.Vout {
 			if vout.ScriptPubKey.Type == "witness_v1_taproot" {
+				// we use the fix sized conversion below with a panic
+				txidBytes, _ := hex.DecodeString(tx.Txid)
+				blockHashBytes, _ := hex.DecodeString(block.Hash)
+
 				value := utils.ConvertFloatBTCtoSats(vout.Value)
 				utxos = append(utxos, &types.UTXO{
-					Txid:         tx.Txid,
+					Txid:         utils.ConvertToFixedLength32(txidBytes),
 					Vout:         vout.N,
 					Value:        value,
 					ScriptPubKey: vout.ScriptPubKey.Hex,
 					BlockHeight:  block.Height,
-					BlockHash:    block.Hash,
+					BlockHash:    utils.ConvertToFixedLength32(blockHashBytes),
 					Timestamp:    block.Timestamp,
 					Spent:        value == 0, // Mark as spent if value is 0
 				})
@@ -72,15 +78,19 @@ func extractSpentTaprootPubKeysFromTx(tx *types.Transaction, block *types.Block)
 					logging.L.Panic().Err(err).Msg("Headers not synced from first taproot like occurrence. Either build complete index or fully sync headers only.")
 					return nil
 				}
-				blockHash = headerInv.Hash
+				blockHash = hex.EncodeToString(headerInv.Hash[:])
 			}
 
+			// we use the fix sized conversion below with a panic
+			txidBytes, _ := hex.DecodeString(vin.Txid)
+			blockHashBytes, _ := hex.DecodeString(blockHash)
+
 			spentUTXOs = append(spentUTXOs, types.UTXO{
-				Txid:         vin.Txid,
+				Txid:         utils.ConvertToFixedLength32(txidBytes),
 				Vout:         vin.Vout,
 				Value:        utils.ConvertFloatBTCtoSats(vin.Prevout.Value),
 				ScriptPubKey: vin.Prevout.ScriptPubKey.Hex,
-				BlockHash:    blockHash,
+				BlockHash:    utils.ConvertToFixedLength32(blockHashBytes),
 				Spent:        true,
 			})
 		} else {

internal/core/routine.go

@@ -1,11 +1,13 @@
 package core
 
 import (
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"time"
 
 	"github.com/setavenger/blindbit-lib/logging"
+	"github.com/setavenger/blindbit-lib/utils"
 	"github.com/setavenger/blindbit-oracle/internal/config"
 	"github.com/setavenger/blindbit-oracle/internal/dblevel"
 	"github.com/setavenger/blindbit-oracle/internal/types"
@@ -62,7 +64,12 @@ func PullBlock(blockHash string) (*types.Block, error) {
 		return nil, fmt.Errorf("block_hash invalid: %s", blockHash)
 	}
 	// this method is preferred over lastHeader because then this function can be called for PreviousBlockHash
-	header, err := dblevel.FetchByBlockHashBlockHeader(blockHash)
+	hashByteSlice, err := hex.DecodeString(blockHash)
+	if err != nil {
+		logging.L.Err(err).Msg("failed to hex decode blockhash")
+		return nil, err
+	}
+	header, err := dblevel.FetchByBlockHashBlockHeader(utils.ConvertToFixedLength32(hashByteSlice))
 	if err != nil && !errors.Is(err, dblevel.NoEntryErr{}) {
 		// we ignore no entry error
 		logging.L.Err(err).Msg("error fetching block header")
@@ -83,6 +90,7 @@ func PullBlock(blockHash string) (*types.Block, error) {
 }
 
 // CheckBlock checks whether the block hash has already been processed and will process the block if needed
+// todo: needs to throw an error
 func CheckBlock(block *types.Block) {
 	// todo add return type error
 	// todo this should fail at the highest instance were its wrapped in,
@@ -116,8 +124,13 @@ func CheckBlock(block *types.Block) {
 		return
 	}
 
+	hashByteSlice, err := hex.DecodeString(block.Hash)
+	if err != nil {
+		logging.L.Err(err).Msg("could not decode blockhash hex")
+		return
+	}
 	err = dblevel.InsertBlockHeaderInv(types.BlockHeaderInv{
-		Hash:   block.Hash,
+		Hash:   utils.ConvertToFixedLength32(hashByteSlice),
 		Height: block.Height,
 		Flag:   true,
 	})
@@ -152,7 +165,7 @@ func HandleBlock(block *types.Block) error {
 		// build map for sorting
 		tweaksForBlockMap := map[string]types.Tweak{}
 		for _, tweak := range tweaksForBlock {
-			tweaksForBlockMap[tweak.Txid] = tweak
+			tweaksForBlockMap[hex.EncodeToString(tweak.Txid[:])] = tweak
 		}
 
 		// we only create one of the two filters no dust can be derived from dust but not vice versa
@@ -161,7 +174,8 @@ func HandleBlock(block *types.Block) error {
 			// full index with dust filter possibility
 			// todo should we sort, overhead created
 			tweakIndexDust := types.TweakIndexDustFromTweakArray(tweaksForBlockMap, block)
-			tweakIndexDust.BlockHash = block.Hash
+			blockHashBytes, _ := hex.DecodeString(block.Hash)
+			tweakIndexDust.BlockHash = utils.ConvertToFixedLength32(blockHashBytes)
 			tweakIndexDust.BlockHeight = block.Height
 
 			err = dblevel.InsertTweakIndexDust(tweakIndexDust)
@@ -173,7 +187,8 @@ func HandleBlock(block *types.Block) error {
 			// normal full index no dust
 			// todo should we sort, overhead created
 			tweakIndex := types.TweakIndexFromTweakArray(tweaksForBlockMap, block)
-			tweakIndex.BlockHash = block.Hash
+			blockHashBytes, _ := hex.DecodeString(block.Hash)
+			tweakIndex.BlockHash = utils.ConvertToFixedLength32(blockHashBytes)
 			tweakIndex.BlockHeight = block.Height
 			err = dblevel.InsertTweakIndex(tweakIndex)
 			if err != nil {
@@ -199,7 +214,7 @@ func HandleBlock(block *types.Block) error {
 	// mark all transaction which have eligible outputs
 	eligibleTransaction := map[string]struct{}{}
 	for _, tweak := range tweaksForBlock {
-		eligibleTransaction[tweak.Txid] = struct{}{}
+		eligibleTransaction[hex.EncodeToString(tweak.Txid[:])] = struct{}{}
 	}
 
 	// first we need to get the new outputs because some of them might/will be spent in the same block

internal/core/spentutxos.go

@@ -21,7 +21,7 @@ func BuildSpentUTXOIndex(utxos []types.UTXO, block *types.Block) (types.SpentOut
 	blockHashBytes = utils.ReverseBytes(blockHashBytes)
 
 	spentOutpointsIndex := types.SpentOutpointsIndex{
-		BlockHash:   block.Hash,
+		BlockHash:   utils.ConvertToFixedLength32(blockHashBytes),
 		BlockHeight: block.Height,
 	}
 

internal/core/sync.go

@@ -1,11 +1,13 @@
 package core
 
 import (
+	"encoding/hex"
 	"errors"
 	"sort"
 	"sync"
 
 	"github.com/setavenger/blindbit-lib/logging"
+	"github.com/setavenger/blindbit-lib/utils"
 	"github.com/setavenger/blindbit-oracle/internal/config"
 	"github.com/setavenger/blindbit-oracle/internal/dblevel"
 	"github.com/setavenger/blindbit-oracle/internal/types"
@@ -288,8 +290,16 @@ func PreSyncHeaders() error {
 		// convert BlockHeaders to BlockerHeadersInv
 		var headersInv []types.BlockHeaderInv
 		for _, header := range headers {
+			blockHashSlice, err := hex.DecodeString(header.Hash)
+			if err != nil {
+				// todo: remove all hex.Decode or hex.Encode code
+				// there are only very few places where this actually needed
+				logging.L.Err(err).Msg("blockhash could not be hex decoded")
+				return err
+			}
+
 			headersInv = append(headersInv, types.BlockHeaderInv{
-				Hash:   header.Hash,
+				Hash:   utils.ConvertToFixedLength32(blockHashSlice),
 				Height: header.Height,
 				Flag:   false,
 			})