sqlite autovacuum and busy_timeout

Author: angaz

cmd/crawler/api.go

@@ -21,8 +21,10 @@ var (
 		Usage:  "API server for the crawler",
 		Action: startAPI,
 		Flags: []cli.Flag{
-			&crawlerDBFlag,
 			&apiDBFlag,
+			&autovacuumFlag,
+			&busyTimeoutFlag,
+			&crawlerDBFlag,
 			&dropNodesTimeFlag,
 		},
 	}
@@ -44,7 +46,14 @@ var (
 )
 
 func startAPI(ctx *cli.Context) error {
-	crawlerDB, err := sql.Open("sqlite", ctx.String(crawlerDBFlag.Name))
+	autovacuum := ctx.String(autovacuumFlag.Name)
+	busyTimeout := ctx.Uint64(busyTimeoutFlag.Name)
+
+	crawlerDB, err := openSQLiteDB(
+		ctx.String(crawlerDBFlag.Name),
+		autovacuum,
+		busyTimeout,
+	)
 	if err != nil {
 		return err
 	}
@@ -54,7 +63,11 @@ func startAPI(ctx *cli.Context) error {
 	if _, err := os.Stat(apiDBPath); os.IsNotExist(err) {
 		shouldInit = true
 	}
-	nodeDB, err := sql.Open("sqlite", apiDBPath)
+	nodeDB, err := openSQLiteDB(
+		apiDBPath,
+		autovacuum,
+		busyTimeout,
+	)
 	if err != nil {
 		return err
 	}

cmd/crawler/crawlercmd.go

@@ -42,19 +42,21 @@ var (
 		Usage:  "Crawl the ethereum network",
 		Action: crawlNodes,
 		Flags: []cli.Flag{
-			utils.GoerliFlag,
-			utils.SepoliaFlag,
-			utils.NetworkIdFlag,
+			&autovacuumFlag,
 			&bootnodesFlag,
-			&nodeURLFlag,
-			&nodeFileFlag,
-			&timeoutFlag,
+			&busyTimeoutFlag,
 			&crawlerDBFlag,
+			&geoipdbFlag,
 			&listenAddrFlag,
-			&nodekeyFlag,
+			&nodeFileFlag,
+			&nodeURLFlag,
 			&nodedbFlag,
-			&geoipdbFlag,
+			&nodekeyFlag,
+			&timeoutFlag,
 			&workersFlag,
+			utils.GoerliFlag,
+			utils.NetworkIdFlag,
+			utils.SepoliaFlag,
 		},
 	}
 	bootnodesFlag = cli.StringSliceFlag{
@@ -118,7 +120,13 @@ func crawlNodes(ctx *cli.Context) error {
 			shouldInit = true
 		}
 		var err error
-		if db, err = sql.Open("sqlite", name); err != nil {
+
+		db, err = openSQLiteDB(
+			name,
+			ctx.String(autovacuumFlag.Name),
+			ctx.Uint64(busyTimeoutFlag.Name),
+		)
+		if err != nil {
 			panic(err)
 		}
 		log.Info("Connected to db")

cmd/crawler/flags.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	autovacuumFlag = cli.StringFlag{
+		Name: "autovacuum",
+		Usage: ("Sets the autovacuum value for the databases. Possible values: " +
+			"NONE, FULL, or INCREMENTAL. " +
+			"https://www.sqlite.org/pragma.html#pragma_auto_vacuum"),
+		Value: "INCREMENTAL",
+	}
+	busyTimeoutFlag = cli.Uint64Flag{
+		Name: "busy-timeout",
+		Usage: ("Sets the busy_timeout value for the database in milliseconds. " +
+			"https://www.sqlite.org/pragma.html#pragma_busy_timeout"),
+		Value: 3000,
+	}
+)

cmd/crawler/setup.go

@@ -77,18 +77,18 @@ var (
 
 // Flags holds all command-line flags required for debugging.
 var Flags = []cli.Flag{
-	&verbosityFlag,
-	&vmoduleFlag,
-	&logjsonFlag,
 	&backtraceAtFlag,
+	&blockprofilerateFlag,
+	&cpuprofileFlag,
 	&debugFlag,
-	&pprofFlag,
+	&logjsonFlag,
+	&memprofilerateFlag,
 	&pprofAddrFlag,
+	&pprofFlag,
 	&pprofPortFlag,
-	&memprofilerateFlag,
-	&blockprofilerateFlag,
-	&cpuprofileFlag,
 	&traceFlag,
+	&verbosityFlag,
+	&vmoduleFlag,
 }
 
 var glogger *log.GlogHandler

cmd/crawler/utils.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func openSQLiteDB(
+	name,
+	autovacuum string,
+	busyTimeout uint64,
+) (*sql.DB, error) {
+	db, err := sql.Open("sqlite", name)
+	if err != nil {
+		return nil, fmt.Errorf("error opening database: %w", err)
+	}
+	_, err = db.Exec("PRAGMA auto_vacuum = " + autovacuum)
+	if err != nil {
+		return nil, fmt.Errorf("error setting auto_vacuum: %w", err)
+	}
+	_, err = db.Exec(fmt.Sprintf("PRAGMA busy_timeout = %d", busyTimeout))
+	if err != nil {
+		return nil, fmt.Errorf("error setting busy_timeout: %w", err)
+	}
+
+	return db, nil
+}