Skip to content

Commit 3ee5bb6

Browse files
sputn1cksputn1ck
committed
loopdb: add postgres store
This commit adds a postgres store to the loopdb package. Ths postgres migrator uses a replacer filesystem to replace the sqlite types to postgres types in the migration.
1 parent ab8923f commit 3ee5bb6

File tree

4 files changed

+300
-0
lines changed

4 files changed

+300
-0
lines changed

‎loopdb/postgres.go‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
package loopdb
2+
3+
import (
4+
"database/sql"
5+
"fmt"
6+
"testing"
7+
"time"
8+
9+
"github.com/btcsuite/btcd/chaincfg"
10+
postgres_migrate "github.com/golang-migrate/migrate/v4/database/postgres"
11+
_ "github.com/golang-migrate/migrate/v4/source/file"
12+
"github.com/lightninglabs/loop/loopdb/sqlc"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
const (
17+
dsnTemplate = "postgres://%v:%v@%v:%d/%v?sslmode=%v"
18+
)
19+
20+
var (
21+
// DefaultPostgresFixtureLifetime is the default maximum time a Postgres
22+
// test fixture is being kept alive. After that time the docker
23+
// container will be terminated forcefully, even if the tests aren't
24+
// fully executed yet. So this time needs to be chosen correctly to be
25+
// longer than the longest expected individual test run time.
26+
DefaultPostgresFixtureLifetime = 10 * time.Minute
27+
)
28+
29+
// PostgresConfig holds the postgres database configuration.
30+
type PostgresConfig struct {
31+
SkipMigrations bool `long:"skipmigrations" description:"Skip applying migrations on startup."`
32+
Host string `long:"host" description:"Database server hostname."`
33+
Port int `long:"port" description:"Database server port."`
34+
User string `long:"user" description:"Database user."`
35+
Password string `long:"password" description:"Database user's password."`
36+
DBName string `long:"dbname" description:"Database name to use."`
37+
MaxOpenConnections int32 `long:"maxconnections" description:"Max open connections to keep alive to the database server."`
38+
RequireSSL bool `long:"requiressl" description:"Whether to require using SSL (mode: require) when connecting to the server."`
39+
}
40+
41+
// DSN returns the dns to connect to the database.
42+
func (s *PostgresConfig) DSN(hidePassword bool) string {
43+
var sslMode = "disable"
44+
if s.RequireSSL {
45+
sslMode = "require"
46+
}
47+
48+
password := s.Password
49+
if hidePassword {
50+
// Placeholder used for logging the DSN safely.
51+
password = "****"
52+
}
53+
54+
return fmt.Sprintf(dsnTemplate, s.User, password, s.Host, s.Port,
55+
s.DBName, sslMode)
56+
}
57+
58+
// PostgresStore is a database store implementation that uses a Postgres
59+
// backend.
60+
type PostgresStore struct {
61+
cfg *PostgresConfig
62+
63+
*BaseDB
64+
}
65+
66+
// NewPostgresStore creates a new store that is backed by a Postgres database
67+
// backend.
68+
func NewPostgresStore(cfg *PostgresConfig,
69+
network *chaincfg.Params) (*PostgresStore, error) {
70+
71+
log.Infof("Using SQL database '%s'", cfg.DSN(true))
72+
73+
rawDb, err := sql.Open("pgx", cfg.DSN(false))
74+
if err != nil {
75+
return nil, err
76+
}
77+
78+
if !cfg.SkipMigrations {
79+
// Now that the database is open, populate the database with
80+
// our set of schemas based on our embedded in-memory file
81+
// system.
82+
//
83+
// First, we'll need to open up a new migration instance for
84+
// our current target database: sqlite.
85+
driver, err := postgres_migrate.WithInstance(
86+
rawDb, &postgres_migrate.Config{},
87+
)
88+
if err != nil {
89+
return nil, err
90+
}
91+
92+
postgresFS := newReplacerFS(sqlSchemas, map[string]string{
93+
"BLOB": "BYTEA",
94+
"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
95+
})
96+
97+
err = applyMigrations(
98+
postgresFS, driver, "sqlc/migrations", cfg.DBName,
99+
)
100+
if err != nil {
101+
return nil, err
102+
}
103+
}
104+
105+
queries := sqlc.New(rawDb)
106+
107+
return &PostgresStore{
108+
cfg: cfg,
109+
BaseDB: &BaseDB{
110+
DB: rawDb,
111+
Queries: queries,
112+
network: network,
113+
},
114+
}, nil
115+
}
116+
117+
// NewTestPostgresDB is a helper function that creates a Postgres database for
118+
// testing.
119+
func NewTestPostgresDB(t *testing.T) *PostgresStore {
120+
t.Helper()
121+
122+
t.Logf("Creating new Postgres DB for testing")
123+
124+
sqlFixture := NewTestPgFixture(t, DefaultPostgresFixtureLifetime)
125+
store, err := NewPostgresStore(
126+
sqlFixture.GetConfig(), &chaincfg.MainNetParams,
127+
)
128+
require.NoError(t, err)
129+
130+
t.Cleanup(func() {
131+
sqlFixture.TearDown(t)
132+
})
133+
134+
return store
135+
}

‎loopdb/postgres_fixture.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package loopdb
2+
3+
import (
4+
"context"
5+
"database/sql"
6+
"fmt"
7+
"strconv"
8+
"strings"
9+
"testing"
10+
"time"
11+
12+
_ "github.com/lib/pq"
13+
"github.com/ory/dockertest/v3"
14+
"github.com/ory/dockertest/v3/docker"
15+
"github.com/stretchr/testify/require"
16+
)
17+
18+
const (
19+
testPgUser = "test"
20+
testPgPass = "test"
21+
testPgDBName = "test"
22+
PostgresTag = "11"
23+
)
24+
25+
// TestPgFixture is a test fixture that starts a Postgres 11 instance in a
26+
// docker container.
27+
type TestPgFixture struct {
28+
db *sql.DB
29+
pool *dockertest.Pool
30+
resource *dockertest.Resource
31+
host string
32+
port int
33+
}
34+
35+
// NewTestPgFixture constructs a new TestPgFixture starting up a docker
36+
// container running Postgres 11. The started container will expire in after
37+
// the passed duration.
38+
func NewTestPgFixture(t *testing.T, expiry time.Duration) *TestPgFixture {
39+
// Use a sensible default on Windows (tcp/http) and linux/osx (socket)
40+
// by specifying an empty endpoint.
41+
pool, err := dockertest.NewPool("")
42+
require.NoError(t, err, "Could not connect to docker")
43+
44+
// Pulls an image, creates a container based on it and runs it.
45+
resource, err := pool.RunWithOptions(&dockertest.RunOptions{
46+
Repository: "postgres",
47+
Tag: PostgresTag,
48+
Env: []string{
49+
fmt.Sprintf("POSTGRES_USER=%v", testPgUser),
50+
fmt.Sprintf("POSTGRES_PASSWORD=%v", testPgPass),
51+
fmt.Sprintf("POSTGRES_DB=%v", testPgDBName),
52+
"listen_addresses='*'",
53+
},
54+
Cmd: []string{
55+
"postgres",
56+
"-c", "log_statement=all",
57+
"-c", "log_destination=stderr",
58+
},
59+
}, func(config *docker.HostConfig) {
60+
// Set AutoRemove to true so that stopped container goes away
61+
// by itself.
62+
config.AutoRemove = true
63+
config.RestartPolicy = docker.RestartPolicy{Name: "no"}
64+
})
65+
require.NoError(t, err, "Could not start resource")
66+
67+
hostAndPort := resource.GetHostPort("5432/tcp")
68+
parts := strings.Split(hostAndPort, ":")
69+
host := parts[0]
70+
port, err := strconv.ParseInt(parts[1], 10, 64)
71+
require.NoError(t, err)
72+
73+
fixture := &TestPgFixture{
74+
host: host,
75+
port: int(port),
76+
}
77+
databaseURL := fixture.GetDSN()
78+
log.Infof("Connecting to Postgres fixture: %v\n", databaseURL)
79+
80+
// Tell docker to hard kill the container in "expiry" seconds.
81+
require.NoError(t, resource.Expire(uint(expiry.Seconds())))
82+
83+
// Exponential backoff-retry, because the application in the container
84+
// might not be ready to accept connections yet.
85+
pool.MaxWait = 120 * time.Second
86+
87+
var testDB *sql.DB
88+
err = pool.Retry(func() error {
89+
testDB, err = sql.Open("postgres", databaseURL)
90+
if err != nil {
91+
return err
92+
}
93+
return testDB.Ping()
94+
})
95+
require.NoError(t, err, "Could not connect to docker")
96+
97+
// Now fill in the rest of the fixture.
98+
fixture.db = testDB
99+
fixture.pool = pool
100+
fixture.resource = resource
101+
102+
return fixture
103+
}
104+
105+
// GetDSN returns the DSN (Data Source Name) for the started Postgres node.
106+
func (f *TestPgFixture) GetDSN() string {
107+
return f.GetConfig().DSN(false)
108+
}
109+
110+
// GetConfig returns the full config of the Postgres node.
111+
func (f *TestPgFixture) GetConfig() *PostgresConfig {
112+
return &PostgresConfig{
113+
Host: f.host,
114+
Port: f.port,
115+
User: testPgUser,
116+
Password: testPgPass,
117+
DBName: testPgDBName,
118+
RequireSSL: false,
119+
}
120+
}
121+
122+
// TearDown stops the underlying docker container.
123+
func (f *TestPgFixture) TearDown(t *testing.T) {
124+
err := f.pool.Purge(f.resource)
125+
require.NoError(t, err, "Could not purge resource")
126+
}
127+
128+
// ClearDB clears the database.
129+
func (f *TestPgFixture) ClearDB(t *testing.T) {
130+
dbConn, err := sql.Open("postgres", f.GetDSN())
131+
require.NoError(t, err)
132+
133+
_, err = dbConn.ExecContext(
134+
context.Background(),
135+
`DROP SCHEMA IF EXISTS public CASCADE;
136+
CREATE SCHEMA public;`,
137+
)
138+
require.NoError(t, err)
139+
}

‎loopdb/test_postgres.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
//go:build test_db_postgres
2+
// +build test_db_postgres
3+
4+
package loopdb
5+
6+
import (
7+
"testing"
8+
)
9+
10+
// NewTestDB is a helper function that creates a Postgres database for testing.
11+
func NewTestDB(t *testing.T) *PostgresStore {
12+
return NewTestPostgresDB(t)
13+
}

‎loopdb/test_sqlite.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
//go:build !test_db_postgres
2+
// +build !test_db_postgres
3+
4+
package loopdb
5+
6+
import (
7+
"testing"
8+
)
9+
10+
// NewTestDB is a helper function that creates an SQLite database for testing.
11+
func NewTestDB(t *testing.T) *SqliteSwapStore {
12+
return NewTestSqliteDB(t)
13+
}

0 commit comments

Comments
 (0)