Major refactoring and libsecp256k1 bindings

We are now using the proper cmd, internal, pkg structure.
The fucntions and information flow within the program will still need to
be overhauled.

Author: setavenger

NOTES.md

@@ -1,3 +1,6 @@
+# possible values: trace, debug, info, warn, error
+log_level = "debug"
+
 # 0.0.0.0:8000 to expose outside of localhost
 # default: "127.0.0.1:8000"
 host = "127.0.0.1:8000"

blindbit.example.toml

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
-	"log"
 	"path"
 
 	"os"
@@ -27,27 +27,41 @@ var (
 )
 
 func init() {
-	flag.StringVar(&config.BaseDirectory, "datadir", config.DefaultBaseDirectory, "Set the base directory for blindbit oracle. Default directory is ~/.blindbit-oracle")
-	flag.BoolVar(&displayVersion, "version", false, "show version of blindbit-oracle")
-	flag.BoolVar(&pruneOnStart, "reprune", false, "set this flag if you want to prune on startup")
-	flag.BoolVar(&exportData, "export-data", false, "export the databases")
+	flag.StringVar(
+		&config.BaseDirectory,
+		"datadir",
+		config.DefaultBaseDirectory,
+		"Set the base directory for blindbit oracle. Default directory is ~/.blindbit-oracle",
+	)
+	flag.BoolVar(
+		&displayVersion,
+		"version",
+		false,
+		"show version of blindbit-oracle",
+	)
+	flag.BoolVar(
+		&pruneOnStart,
+		"reprune",
+		false,
+		"set this flag if you want to prune on startup",
+	)
+	flag.BoolVar(
+		&exportData,
+		"export-data",
+		false,
+		"export the databases",
+	)
 	flag.Parse()
 
 	if displayVersion {
 		// we only need the version for this
 		return
 	}
+
 	config.SetDirectories() // todo a proper set settings function which does it all would be good to avoid several small function calls
 	err := os.Mkdir(config.BaseDirectory, 0750)
-	if err != nil && !strings.Contains(err.Error(), "file exists") {
-		fmt.Println(err.Error())
-		log.Fatal(err)
-	}
-
-	err = os.Mkdir(config.LogsPath, 0750)
-	if err != nil && !strings.Contains(err.Error(), "file exists") {
-		fmt.Println(err.Error())
-		log.Fatal(err)
+	if err != nil && !errors.Is(err, os.ErrExist) {
+		logging.L.Fatal().Err(err).Msg("error creating base directory")
 	}
 
 	logging.L.Info().Msgf("base directory %s", config.BaseDirectory)
@@ -59,7 +73,7 @@ func init() {
 	err = os.Mkdir(config.DBPath, 0750)
 	if err != nil && !strings.Contains(err.Error(), "file exists") {
 		logging.L.Err(err).Msg("error creating db path")
-		panic(err)
+		os.Exit(1)
 	}
 
 	// open levelDB connections
@@ -68,23 +82,23 @@ func init() {
 	if config.CookiePath != "" {
 		data, err := os.ReadFile(config.CookiePath)
 		if err != nil {
-			panic(err)
+			logging.L.Fatal().Err(err).Msg("error reading cookie file")
 		}
 
 		credentials := strings.Split(string(data), ":")
 		if len(credentials) != 2 {
-			panic("cookie file is invalid")
+			logging.L.Fatal().Msg("cookie file is invalid")
 		}
 		config.RpcUser = credentials[0]
 		config.RpcPass = credentials[1]
 	}
 
 	if config.RpcUser == "" {
-		panic("rpc user not set") // todo use cookie file to circumvent this requirement
+		logging.L.Fatal().Msg("rpc user not set") // todo use cookie file to circumvent this requirement
 	}
 
 	if config.RpcPass == "" {
-		panic("rpc pass not set") // todo use cookie file to circumvent this requirement
+		logging.L.Fatal().Msg("rpc pass not set") // todo use cookie file to circumvent this requirement
 	}
 }
 
@@ -96,7 +110,6 @@ func main() {
 	defer logging.L.Info().Msg("Program shut down")
 	defer dblevel.CloseDBs()
 
-	//log.SetFlags(log.LstdFlags | log.Lshortfile | log.Lmicroseconds)
 	interrupt := make(chan os.Signal, 1)
 	signal.Notify(interrupt, os.Interrupt)
 
@@ -108,7 +121,8 @@ func main() {
 
 	if exportData {
 		logging.L.Info().Msg("Exporting data")
-		dataexport.ExportUTXOs(fmt.Sprintf("%s/export/utxos.csv", config.BaseDirectory))
+		dataexport.ExportAll()
+		// dataexport.ExportUTXOs(fmt.Sprintf("%s/export/utxos.csv", config.BaseDirectory))
 		return
 	}
 
@@ -141,10 +155,9 @@ func main() {
 		// only call this if you need to reindex. It doesn't delete anything but takes a couple of minutes to finish
 		//err := core.ReindexDustLimitsOnly()
 		//if err != nil {
-		//	common.ErrorLogger.Fatalln(err)
+		//	logging.L.Err(err).Msg("error reindexing dust limits")
 		//	return
 		//}
-
 	}()
 
 	for {

cmd/blindbit-oracle/main.go

@@ -33,6 +33,7 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.24.0 // indirect
+	github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 // indirect
 	github.com/goccy/go-json v0.10.4 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect

go.mod

@@ -70,6 +70,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg=
 github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ=
+github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI=
 github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM=
 github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=

go.sum

@@ -3,6 +3,7 @@ package config
 import (
 	"errors"
 
+	"github.com/rs/zerolog"
 	"github.com/setavenger/blindbit-lib/logging"
 	"github.com/spf13/viper"
 )
@@ -30,7 +31,7 @@ func LoadConfigs(pathToConfig string) {
 	viper.SetDefault("tweaks_full_basic", true)
 	viper.SetDefault("tweaks_full_with_dust_filter", false)
 	viper.SetDefault("tweaks_cut_through_with_dust_filter", false)
-
+	viper.SetDefault("log_level", "info")
 	// Bind viper keys to environment variables (optional, for backup)
 	viper.AutomaticEnv()
 	viper.BindEnv("host", "HOST")
@@ -46,12 +47,13 @@ func LoadConfigs(pathToConfig string) {
 	viper.BindEnv("tweaks_full_basic", "TWEAKS_FULL_BASIC")
 	viper.BindEnv("tweaks_full_with_dust_filter", "TWEAKS_FULL_WITH_DUST_FILTER")
 	viper.BindEnv("tweaks_cut_through_with_dust_filter", "TWEAKS_CUT_THROUGH_WITH_DUST_FILTER")
+	viper.BindEnv("log_level", "LOG_LEVEL")
 
 	/* read and set config variables */
 	// General
 	SyncStartHeight = viper.GetUint32("sync_start_height")
 	Host = viper.GetString("host")
-
+	LogLevel = viper.GetString("log_level")
 	// Performance
 	MaxParallelRequests = viper.GetUint16("max_parallel_requests")
 	MaxParallelTweakComputations = viper.GetInt("max_parallel_tweak_computations")
@@ -80,7 +82,21 @@ func LoadConfigs(pathToConfig string) {
 	case "testnet":
 		Chain = Testnet3
 	default:
-		panic("chain undefined")
+		logging.L.Fatal().Msg("chain undefined")
+		return
+	}
+
+	switch LogLevel {
+	case "trace":
+		logging.SetLogLevel(zerolog.TraceLevel)
+	case "info":
+		logging.SetLogLevel(zerolog.InfoLevel)
+	case "debug":
+		logging.SetLogLevel(zerolog.DebugLevel)
+	case "warn":
+		logging.SetLogLevel(zerolog.WarnLevel)
+	case "error":
+		logging.SetLogLevel(zerolog.ErrorLevel)
 	}
 
 	// todo print settings

internal/config/config.go

@@ -1,12 +1,19 @@
 package config
 
-import "github.com/setavenger/blindbit-lib/utils"
+import (
+	"github.com/setavenger/blindbit-lib/logging"
+	"github.com/setavenger/blindbit-lib/utils"
+)
 
 // TaprootActivation
 // todo might be inapplicable due to transactions that have taproot prevouts from before the activation
 //
 //	is relevant for the height-to-hash lookup in the db
 
+var (
+	LogLevel = "info"
+)
+
 const (
 	TaprootActivation    uint32 = 709632
 	ConfigFileName       string = "blindbit.toml"
@@ -63,7 +70,7 @@ var (
 	PruneFrequency = 72
 )
 
-// one has to call SetDirectories otherwise common.DBPath will be empty
+// one has to call SetDirectories otherwise config.DBPath will be empty
 var (
 	DBPathHeaders              string
 	DBPathHeadersInv           string // for height to blockHash mapping
@@ -99,6 +106,7 @@ func SetDirectories() {
 func HeaderMustSyncHeight() uint32 {
 	switch Chain {
 	case Mainnet:
+		// height based on heuristic checks to see where no old taproot style coins were locked
 		return 500_000
 	case Signet:
 		return 1
@@ -107,7 +115,8 @@ func HeaderMustSyncHeight() uint32 {
 	case Testnet3:
 		return 1
 	case Unknown:
-		panic("chain not defined")
+		logging.L.Panic().Msg("chain not defined")
+		return 0
 	default:
 		return 1
 	}
@@ -124,7 +133,8 @@ func ChainToString(c chain) string {
 	case Testnet3:
 		return "testnet"
 	default:
-		panic("chain not defined")
+		logging.L.Panic().Msg("chain not defined")
+		return ""
 	}
 
 }

internal/config/vars.go

@@ -23,13 +23,13 @@ func TestBlockAnalysis(t *testing.T) {
 	}
 
 	for _, tweak := range tweaks {
-		logging.L.Info().Bytes("tweak", tweak.TweakData).Str("txid", tweak.Txid).Msg("tweak")
+		logging.L.Info().Hex("tweak", tweak.TweakData[:]).Str("txid", tweak.Txid).Msg("tweak")
 	}
 
 	for _, tx := range block.Txs {
 		for _, tweak := range tweaks {
 			if tx.Txid == tweak.Txid {
-				logging.L.Info().Hex("tweak", tweak.TweakData).Msg("tweak")
+				logging.L.Info().Hex("tweak", tweak.TweakData[:]).Msg("tweak")
 			}
 		}
 	}

internal/core/block_test.go

@@ -4,13 +4,13 @@ import (
 	"errors"
 
 	"github.com/setavenger/blindbit-lib/logging"
-	config "github.com/setavenger/blindbit-oracle/internal/conig"
+	"github.com/setavenger/blindbit-oracle/internal/config"
 	"github.com/setavenger/blindbit-oracle/internal/dblevel"
 	"github.com/setavenger/blindbit-oracle/internal/types"
 )
 
 func overwriteUTXOsWithLookUp(utxos []types.UTXO) error {
-	logging.L.Info().Msg("overwriting utxos with lookup")
+	logging.L.Trace().Msg("overwriting utxos with lookup")
 	var utxosToOverwrite []*types.UTXO
 
 	for _, utxo := range utxos {
@@ -56,7 +56,7 @@ func overwriteUTXOsWithLookUp(utxos []types.UTXO) error {
 
 // todo construct the subsequent deletion of all utxos per transaction once all per transaction are spent
 func markSpentUTXOsAndTweaks(utxos []types.UTXO) error {
-	logging.L.Info().Msg("marking utxos")
+	logging.L.Trace().Msg("marking utxos")
 	if len(utxos) == 0 {
 		if config.Chain == config.Mainnet {
 			// no warnings on other chains as it is very likely to not have any taproot outputs for several blocks on end