database + migrations (#39)

Author: metachris

README.md

@@ -16,6 +16,7 @@ Toolbox and building blocks for new Go projects, to get started quickly and righ
   * Using https://pkg.go.dev/github.com/go-chi/chi/v5 for routing
   * [Urfave](https://cli.urfave.org/) for cli args
 * https://github.com/uber-go/nilaway
+* Postgres database with migrations
 * See also:
   * Public project setup: https://github.com/flashbots/flashbots-repository-template
   * Repository for common Go utilities: https://github.com/flashbots/go-utils
@@ -55,3 +56,19 @@ make lint
 make test
 make fmt
 ```
+
+
+**Database tests (using a live Postgres instance)**
+
+Database tests will be run if the `RUN_DB_TESTS` environment variable is set to `1`.
+
+```bash
+# start the database
+docker run -d --name postgres-test -p 5432:5432 -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=postgres postgres
+
+# run the tests
+RUN_DB_TESTS=1 make test
+
+# stop the database
+docker rm -f postgres-test
+```

common/utils.go

@@ -0,0 +1,10 @@
+package common
+
+import "os"
+
+func GetEnv(key, defaultValue string) string {
+	if value, ok := os.LookupEnv(key); ok {
+		return value
+	}
+	return defaultValue
+}

database/database.go

@@ -0,0 +1,54 @@
+// Package database exposes the postgres database
+package database
+
+import (
+	"os"
+
+	"github.com/flashbots/go-template/database/migrations"
+	"github.com/flashbots/go-template/database/vars"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+type DatabaseService struct {
+	DB *sqlx.DB
+}
+
+func NewDatabaseService(dsn string) (*DatabaseService, error) {
+	db, err := sqlx.Connect("postgres", dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	db.DB.SetMaxOpenConns(50)
+	db.DB.SetMaxIdleConns(10)
+	db.DB.SetConnMaxIdleTime(0)
+
+	if os.Getenv("DB_DONT_APPLY_SCHEMA") == "" {
+		migrate.SetTable(vars.TableMigrations)
+		_, err := migrate.Exec(db.DB, "postgres", migrations.Migrations, migrate.Up)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	dbService := &DatabaseService{DB: db} //nolint:exhaustruct
+	err = dbService.prepareNamedQueries()
+	return dbService, err
+}
+
+func (s *DatabaseService) prepareNamedQueries() (err error) {
+	return nil
+}
+
+func (s *DatabaseService) Close() error {
+	return s.DB.Close()
+}
+
+func (s *DatabaseService) SomeQuery() (count uint64, err error) {
+	query := `SELECT COUNT(*) FROM ` + vars.TableTest + `;`
+	row := s.DB.QueryRow(query)
+	err = row.Scan(&count)
+	return count, err
+}

database/database_test.go

@@ -0,0 +1,50 @@
+package database
+
+import (
+	"os"
+	"testing"
+
+	"github.com/flashbots/go-template/common"
+	"github.com/flashbots/go-template/database/migrations"
+	"github.com/flashbots/go-template/database/vars"
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	runDBTests = os.Getenv("RUN_DB_TESTS") == "1" //|| true
+	testDBDSN  = common.GetEnv("TEST_DB_DSN", "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable")
+)
+
+func resetDatabase(t *testing.T) *DatabaseService {
+	t.Helper()
+	if !runDBTests {
+		t.Skip("Skipping database tests")
+	}
+
+	// Wipe test database
+	_db, err := sqlx.Connect("postgres", testDBDSN)
+	require.NoError(t, err)
+	_, err = _db.Exec(`DROP SCHEMA public CASCADE; CREATE SCHEMA public;`)
+	require.NoError(t, err)
+
+	db, err := NewDatabaseService(testDBDSN)
+	require.NoError(t, err)
+	return db
+}
+
+func TestMigrations(t *testing.T) {
+	db := resetDatabase(t)
+	query := `SELECT COUNT(*) FROM ` + vars.TableMigrations + `;`
+	rowCount := 0
+	err := db.DB.QueryRow(query).Scan(&rowCount)
+	require.NoError(t, err)
+	require.Len(t, migrations.Migrations.Migrations, rowCount)
+}
+
+func Test_DB1(t *testing.T) {
+	db := resetDatabase(t)
+	x, err := db.SomeQuery()
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), x)
+}

database/migrations/001_init_database.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/flashbots/go-template/database/vars"
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+var Migration001InitDatabase = &migrate.Migration{
+	Id: "001-init-database",
+	Up: []string{`
+		CREATE TABLE IF NOT EXISTS ` + vars.TableTest + ` (
+			id          bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+			inserted_at timestamp NOT NULL default current_timestamp
+		);
+	`},
+	Down: []string{`
+		DROP TABLE IF EXISTS ` + vars.TableTest + `;
+	`},
+	DisableTransactionUp:   false,
+	DisableTransactionDown: false,
+}

database/migrations/migration.go

@@ -0,0 +1,12 @@
+// Package migrations contains all the migration files
+package migrations
+
+import (
+	migrate "github.com/rubenv/sql-migrate"
+)
+
+var Migrations = migrate.MemoryMigrationSource{
+	Migrations: []*migrate.Migration{
+		Migration001InitDatabase,
+	},
+}

database/types.go

@@ -0,0 +1,29 @@
+package database
+
+import (
+	"database/sql"
+	"time"
+)
+
+func NewNullInt64(i int64) sql.NullInt64 {
+	return sql.NullInt64{
+		Int64: i,
+		Valid: true,
+	}
+}
+
+func NewNullString(s string) sql.NullString {
+	return sql.NullString{
+		String: s,
+		Valid:  true,
+	}
+}
+
+// NewNullTime returns a sql.NullTime with the given time.Time. If the time is
+// the zero value, the NullTime is invalid.
+func NewNullTime(t time.Time) sql.NullTime {
+	return sql.NullTime{
+		Time:  t,
+		Valid: t != time.Time{},
+	}
+}

database/types_test.go

@@ -0,0 +1,19 @@
+package database
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewNullTime(t *testing.T) {
+	var t1 time.Time
+	nt1 := NewNullTime(t1)
+	require.False(t, nt1.Valid)
+
+	t1 = time.Now()
+	nt1 = NewNullTime(t1)
+	require.True(t, nt1.Valid)
+	require.Equal(t, t1, nt1.Time)
+}

database/vars/tables.go

@@ -0,0 +1,11 @@
+// Package vars contains the database variables such as dynamic table names
+package vars
+
+import "github.com/flashbots/go-template/common"
+
+var (
+	tablePrefix = common.GetEnv("DB_TABLE_PREFIX", "dev")
+
+	TableMigrations = tablePrefix + "_migrations"
+	TableTest       = tablePrefix + "_test"
+)

go.mod

@@ -6,7 +6,10 @@ require (
 	github.com/flashbots/go-utils v0.6.1-0.20240610084140-4461ab748667
 	github.com/go-chi/chi/v5 v5.0.12
 	github.com/google/uuid v1.6.0
+	github.com/jmoiron/sqlx v1.4.0
+	github.com/lib/pq v1.10.9
 	github.com/prometheus/client_golang v1.19.1
+	github.com/rubenv/sql-migrate v1.7.0
 	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.2
 	go.opentelemetry.io/contrib/instrumentation/runtime v0.46.1
@@ -22,6 +25,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ethereum/go-ethereum v1.13.14 // indirect
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/holiman/uint256 v1.2.4 // indirect