major refactor

Author: i-norden

cmd/geth/config.go

@@ -23,8 +23,11 @@ import (
 	"math/big"
 	"os"
 	"reflect"
+	"time"
 	"unicode"
 
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/statediff"
 	"gopkg.in/urfave/cli.v1"
@@ -182,27 +185,48 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	}
 
 	if ctx.GlobalBool(utils.StateDiffFlag.Name) {
-		var dbParams *statediff.DBParams
-		if ctx.GlobalIsSet(utils.StateDiffDBFlag.Name) {
-			dbParams = new(statediff.DBParams)
-			dbParams.ConnectionURL = ctx.GlobalString(utils.StateDiffDBFlag.Name)
+		var dbConfig *sql.Config
+		if ctx.GlobalIsSet(utils.StateDiffWritingFlag.Name) {
+			dbConfig = new(sql.Config)
+			dbConfig.Hostname = ctx.GlobalString(utils.StateDiffDBHostFlag.Name)
+			dbConfig.Port = ctx.GlobalInt(utils.StateDiffDBPortFlag.Name)
+			dbConfig.DatabaseName = ctx.GlobalString(utils.StateDiffDBNameFlag.Name)
+			dbConfig.Username = ctx.GlobalString(utils.StateDiffDBUserFlag.Name)
+			dbConfig.Password = ctx.GlobalString(utils.StateDiffDBPasswordFlag.Name)
+
 			if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) {
-				dbParams.ID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name)
+				dbConfig.ID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name)
 			} else {
 				utils.Fatalf("Must specify node ID for statediff DB output")
 			}
+
 			if ctx.GlobalIsSet(utils.StateDiffDBClientNameFlag.Name) {
-				dbParams.ClientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name)
+				dbConfig.ClientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name)
 			} else {
 				utils.Fatalf("Must specify client name for statediff DB output")
 			}
-		} else {
-			if ctx.GlobalBool(utils.StateDiffWritingFlag.Name) {
-				utils.Fatalf("Must pass DB parameters if enabling statediff write loop")
+
+			if ctx.GlobalIsSet(utils.StateDiffDBMinConns.Name) {
+				dbConfig.MinConns = ctx.GlobalInt(utils.StateDiffDBMinConns.Name)
+			}
+			if ctx.GlobalIsSet(utils.StateDiffDBMaxConns.Name) {
+				dbConfig.MaxConns = ctx.GlobalInt(utils.StateDiffDBMaxConns.Name)
+			}
+			if ctx.GlobalIsSet(utils.StateDiffDBMaxIdleConns.Name) {
+				dbConfig.MaxIdle = ctx.GlobalInt(utils.StateDiffDBMaxIdleConns.Name)
+			}
+			if ctx.GlobalIsSet(utils.StateDiffDBMaxConnLifetime.Name) {
+				dbConfig.MaxConnLifetime = ctx.GlobalDuration(utils.StateDiffDBMaxConnLifetime.Name) * time.Second
+			}
+			if ctx.GlobalIsSet(utils.StateDiffDBMaxConnIdleTime.Name) {
+				dbConfig.MaxConnIdleTime = ctx.GlobalDuration(utils.StateDiffDBMaxConnIdleTime.Name) * time.Second
+			}
+			if ctx.GlobalIsSet(utils.StateDiffDBConnTimeout.Name) {
+				dbConfig.ConnTimeout = ctx.GlobalDuration(utils.StateDiffDBConnTimeout.Name) * time.Second
 			}
 		}
 		p := statediff.ServiceParams{
-			DBParams:        dbParams,
+			DBParams:        dbConfig,
 			EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name),
 			NumWorkers:      ctx.GlobalUint(utils.StateDiffWorkersFlag.Name),
 		}

cmd/geth/main.go

@@ -149,7 +149,17 @@ var (
 		utils.GpoIgnoreGasPriceFlag,
 		utils.MinerNotifyFullFlag,
 		utils.StateDiffFlag,
-		utils.StateDiffDBFlag,
+		utils.StateDiffDBNameFlag,
+		utils.StateDiffDBPasswordFlag,
+		utils.StateDiffDBUserFlag,
+		utils.StateDiffDBHostFlag,
+		utils.StateDiffDBPortFlag,
+		utils.StateDiffDBMaxConnLifetime,
+		utils.StateDiffDBMaxConnIdleTime,
+		utils.StateDiffDBMaxConns,
+		utils.StateDiffDBMinConns,
+		utils.StateDiffDBMaxIdleConns,
+		utils.StateDiffDBConnTimeout,
 		utils.StateDiffDBNodeIDFlag,
 		utils.StateDiffDBClientNameFlag,
 		utils.StateDiffWritingFlag,

cmd/geth/usage.go

@@ -225,7 +225,17 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 		Name: "STATE DIFF",
 		Flags: []cli.Flag{
 			utils.StateDiffFlag,
-			utils.StateDiffDBFlag,
+			utils.StateDiffDBNameFlag,
+			utils.StateDiffDBPasswordFlag,
+			utils.StateDiffDBUserFlag,
+			utils.StateDiffDBHostFlag,
+			utils.StateDiffDBPortFlag,
+			utils.StateDiffDBMaxConnLifetime,
+			utils.StateDiffDBMaxConnIdleTime,
+			utils.StateDiffDBMaxConns,
+			utils.StateDiffDBMinConns,
+			utils.StateDiffDBMaxIdleConns,
+			utils.StateDiffDBConnTimeout,
 			utils.StateDiffDBNodeIDFlag,
 			utils.StateDiffDBClientNameFlag,
 			utils.StateDiffWritingFlag,

cmd/utils/flags.go

@@ -786,16 +786,59 @@ var (
 		Name:  "statediff",
 		Usage: "Enables the processing of state diffs between each block",
 	}
-	StateDiffDBFlag = cli.StringFlag{
-		Name:  "statediff.db",
-		Usage: "PostgreSQL database connection string for writing state diffs",
+	StateDiffDBHostFlag = cli.StringFlag{
+		Name:  "statediff.db.host",
+		Usage: "Statediff database hostname/ip",
+		Value: "localhost",
+	}
+	StateDiffDBPortFlag = cli.IntFlag{
+		Name:  "statediff.db.port",
+		Usage: "Statediff database port",
+		Value: 5432,
+	}
+	StateDiffDBNameFlag = cli.StringFlag{
+		Name:  "statediff.db.name",
+		Usage: "Statediff database name",
+	}
+	StateDiffDBPasswordFlag = cli.StringFlag{
+		Name:  "statediff.db.password",
+		Usage: "Statediff database password",
+	}
+	StateDiffDBUserFlag = cli.StringFlag{
+		Name:  "statediff.db.user",
+		Usage: "Statediff database username",
+		Value: "postgres",
+	}
+	StateDiffDBMaxConnLifetime = cli.DurationFlag{
+		Name:  "statediff.db.maxconnlifetime",
+		Usage: "Statediff database maximum connection lifetime (in seconds)",
+	}
+	StateDiffDBMaxConnIdleTime = cli.DurationFlag{
+		Name:  "statediff.db.maxconnidletime",
+		Usage: "Statediff database maximum connection idle time (in seconds)",
+	}
+	StateDiffDBMaxConns = cli.IntFlag{
+		Name:  "statediff.db.maxconns",
+		Usage: "Statediff database maximum connections",
+	}
+	StateDiffDBMinConns = cli.IntFlag{
+		Name:  "statediff.db.minconns",
+		Usage: "Statediff database minimum connections",
+	}
+	StateDiffDBMaxIdleConns = cli.IntFlag{
+		Name:  "statediff.db.maxidleconns",
+		Usage: "Statediff database maximum idle connections",
+	}
+	StateDiffDBConnTimeout = cli.DurationFlag{
+		Name:  "statediff.db.conntimeout",
+		Usage: "Statediff database connection timeout (in seconds)",
 	}
 	StateDiffDBNodeIDFlag = cli.StringFlag{
-		Name:  "statediff.dbnodeid",
+		Name:  "statediff.db.nodeid",
 		Usage: "Node ID to use when writing state diffs to database",
 	}
 	StateDiffDBClientNameFlag = cli.StringFlag{
-		Name:  "statediff.dbclientname",
+		Name:  "statediff.db.clientname",
 		Usage: "Client name to use when writing state diffs to database",
 	}
 	StateDiffWritingFlag = cli.BoolFlag{

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff
+	github.com/georgysavva/scany v0.2.9
 	github.com/go-ole/go-ole v1.2.1 // indirect
 	github.com/go-stack/stack v1.8.0
 	github.com/golang/protobuf v1.4.3
@@ -46,6 +47,10 @@ require (
 	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
 	github.com/ipfs/go-ipld-format v0.2.0
+	github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect
+	github.com/jackc/pgconn v1.10.0
+	github.com/jackc/pgx v3.6.2+incompatible
+	github.com/jackc/pgx/v4 v4.13.0
 	github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458
 	github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e
 	github.com/jmoiron/sqlx v1.2.0
@@ -55,7 +60,6 @@ require (
 	github.com/lib/pq v1.10.2
 	github.com/mattn/go-colorable v0.1.8
 	github.com/mattn/go-isatty v0.0.12
-	github.com/mattn/go-sqlite3 v1.14.7 // indirect
 	github.com/multiformats/go-multihash v0.0.14
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416
@@ -70,7 +74,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
-	golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
+	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912
 	golang.org/x/text v0.3.6