diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 62393b938..a3442c8a5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -275,6 +275,17 @@ jobs: itest: name: integration test runs-on: ubuntu-latest + strategy: + # Allow other tests in the matrix to continue if one fails. + fail-fast: false + matrix: + include: + - name: bbolt + args: dbbackend=bbolt + - name: sqlite + args: dbbackend=sqlite + - name: postgres + args: dbbackend=postgres steps: - name: git checkout uses: actions/checkout@v4 @@ -295,8 +306,8 @@ jobs: working-directory: ./app run: yarn - - name: run check - run: make itest + - name: run itest ${{ matrix.name }} + run: make itest ${{ matrix.args }} - name: Zip log files on failure if: ${{ failure() }} diff --git a/.golangci.yml b/.golangci.yml index 25bf2ec12..62cf7fa17 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,6 +11,7 @@ run: - watchtowerrpc - neutrinorpc - peersrpc + - dev linters-settings: govet: diff --git a/config.go b/config.go index ec800a2e9..4e201f27f 100644 --- a/config.go +++ b/config.go @@ -237,6 +237,13 @@ type Config struct { // over an in-memory connection on startup. This is only set in // integrated lnd mode. lndAdminMacaroon []byte + + // DevConfig is a config struct that is empty if lit is built without + // the `dev` flag (in other words when it is build for a production + // environment). This allows us to have config values that are then + // only available in development mode which lets us run itests against + // features not yet available in production. + *DevConfig } // lndConnectParams returns the connection parameters to connect to the local @@ -337,8 +344,9 @@ func defaultConfig() *Config { Autopilot: &autopilotserver.Config{ PingCadence: time.Hour, }, - Firewall: firewall.DefaultConfig(), - Accounts: &accounts.Config{}, + Firewall: firewall.DefaultConfig(), + Accounts: &accounts.Config{}, + DevConfig: defaultDevConfig(), } } @@ -467,6 +475,11 @@ func loadAndValidateConfig(interceptor signal.Interceptor) (*Config, error) { ) } + err = cfg.DevConfig.Validate(litDir, cfg.Network) + if err != nil { + return nil, err + } + // Initiate our listeners. For now, we only support listening on one // port at a time because we can only pass in one pre-configured RPC // listener into lnd. diff --git a/config_dev.go b/config_dev.go new file mode 100644 index 000000000..28357852c --- /dev/null +++ b/config_dev.go @@ -0,0 +1,117 @@ +//go:build dev + +package terminal + +import ( + "path/filepath" + + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/clock" +) + +const ( + // DatabaseBackendSqlite is the name of the SQLite database backend. + DatabaseBackendSqlite = "sqlite" + + // DatabaseBackendPostgres is the name of the Postgres database backend. + DatabaseBackendPostgres = "postgres" + + // DatabaseBackendBbolt is the name of the bbolt database backend. + DatabaseBackendBbolt = "bbolt" + + // defaultSqliteDatabaseFileName is the default name of the SQLite + // database file. + defaultSqliteDatabaseFileName = "litd.db" +) + +// defaultSqliteDatabasePath is the default path under which we store +// the SQLite database file. +var defaultSqliteDatabasePath = filepath.Join( + DefaultLitDir, DefaultNetwork, defaultSqliteDatabaseFileName, +) + +// DevConfig is a struct that holds the configuration options for a development +// environment. The purpose of this struct is to hold config options for +// features not yet available in production. Since our itests are built with +// the dev tag, we can test these features in our itests. +// +// nolint:lll +type DevConfig struct { + // DatabaseBackend is the database backend we will use for storing all + // account related data. While this feature is still in development, we + // include the bbolt type here so that our itests can continue to be + // tested against a bbolt backend. Once the full bbolt to SQL migration + // is complete, however, we will remove the bbolt option. + DatabaseBackend string `long:"databasebackend" description:"The database backend to use for storing all account related data." choice:"bbolt" choice:"sqlite" choice:"postgres"` + + // Sqlite holds the configuration options for a SQLite database + // backend. + Sqlite *db.SqliteConfig `group:"sqlite" namespace:"sqlite"` + + // Postgres holds the configuration options for a Postgres database + Postgres *db.PostgresConfig `group:"postgres" namespace:"postgres"` +} + +// Validate checks that all the values set in our DevConfig are valid and uses +// the passed parameters to override any defaults if necessary. +func (c *DevConfig) Validate(dbDir, network string) error { + // We'll update the database file location if it wasn't set. + if c.Sqlite.DatabaseFileName == defaultSqliteDatabasePath { + c.Sqlite.DatabaseFileName = filepath.Join( + dbDir, network, defaultSqliteDatabaseFileName, + ) + } + + return nil +} + +// defaultDevConfig returns a new DevConfig with default values set. +func defaultDevConfig() *DevConfig { + return &DevConfig{ + Sqlite: &db.SqliteConfig{ + DatabaseFileName: defaultSqliteDatabasePath, + }, + Postgres: &db.PostgresConfig{ + Host: "localhost", + Port: 5432, + MaxOpenConnections: 10, + }, + } +} + +// NewAccountStore creates a new account store based on the chosen database +// backend. +func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) { + switch cfg.DatabaseBackend { + case DatabaseBackendSqlite: + // Before we initialize the SQLite store, we'll make sure that + // the directory where we will store the database file exists. + networkDir := filepath.Join(cfg.LitDir, cfg.Network) + err := makeDirectories(networkDir) + if err != nil { + return nil, err + } + + sqlStore, err := db.NewSqliteStore(cfg.Sqlite) + if err != nil { + return nil, err + } + + return accounts.NewSQLStore(sqlStore.BaseDB, clock), nil + + case DatabaseBackendPostgres: + sqlStore, err := db.NewPostgresStore(cfg.Postgres) + if err != nil { + return nil, err + } + + return accounts.NewSQLStore(sqlStore.BaseDB, clock), nil + + default: + return accounts.NewBoltStore( + filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, + clock, + ) + } +} diff --git a/config_prod.go b/config_prod.go new file mode 100644 index 000000000..3bb64e746 --- /dev/null +++ b/config_prod.go @@ -0,0 +1,33 @@ +//go:build !dev + +package terminal + +import ( + "path/filepath" + + "github.com/lightninglabs/lightning-terminal/accounts" + "github.com/lightningnetwork/lnd/clock" +) + +// DevConfig is an empty shell struct that allows us to build without the dev +// tag. This struct is embedded in the main Config struct, and it adds no new +// functionality in a production build. +type DevConfig struct{} + +// defaultDevConfig returns an empty DevConfig struct. +func defaultDevConfig() *DevConfig { + return &DevConfig{} +} + +// Validate is a no-op function during a production build. +func (c *DevConfig) Validate(_, _ string) error { + return nil +} + +// NewAccountStore creates a new account store using the default Bolt backend +// since in production, this is the only backend supported currently. +func NewAccountStore(cfg *Config, clock clock.Clock) (accounts.Store, error) { + return accounts.NewBoltStore( + filepath.Dir(cfg.MacaroonPath), accounts.DBFilename, clock, + ) +} diff --git a/itest/litd_node.go b/itest/litd_node.go index e9be55972..48cd6353a 100644 --- a/itest/litd_node.go +++ b/itest/litd_node.go @@ -8,6 +8,7 @@ import ( "encoding/hex" "encoding/json" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -25,6 +26,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/lightninglabs/faraday/frdrpc" terminal "github.com/lightninglabs/lightning-terminal" + "github.com/lightninglabs/lightning-terminal/db" "github.com/lightninglabs/lightning-terminal/litrpc" "github.com/lightninglabs/lightning-terminal/subservers" "github.com/lightninglabs/loop/looprpc" @@ -60,7 +62,12 @@ var ( numActiveNodes = 0 numActiveNodesMtx sync.Mutex - defaultLndPassphrase = []byte("default-wallet-password") + // litDBBackend is a command line flag for specifying the database + // backend to use when starting a LiT daemon. + litDBBackend = flag.String( + "litdbbackend", terminal.DatabaseBackendBbolt, "Set the "+ + "database backend to use when starting a LiT daemon.", + ) ) type LitNodeConfig struct { @@ -80,6 +87,9 @@ type LitNodeConfig struct { LitTLSCertPath string LitMacPath string + DBBackend string + PostgresConfig *db.PostgresConfig + UIPassword string LitDir string FaradayDir string @@ -220,8 +230,20 @@ func (cfg *LitNodeConfig) defaultLitdArgs() *litArgs { "restcors": "*", "lnd.debuglevel": "trace,GRPC=error,PEER=info", "lndconnectinterval": "200ms", + "databasebackend": cfg.DBBackend, } ) + + if cfg.DBBackend == terminal.DatabaseBackendPostgres { + args["postgres.host"] = cfg.PostgresConfig.Host + args["postgres.port"] = fmt.Sprintf( + "%d", cfg.PostgresConfig.Port, + ) + args["postgres.user"] = cfg.PostgresConfig.User + args["postgres.password"] = cfg.PostgresConfig.Password + args["postgres.dbname"] = cfg.PostgresConfig.DBName + } + for _, arg := range cfg.LitArgs { parts := strings.Split(arg, "=") option := strings.TrimLeft(parts[0], "--") @@ -417,6 +439,28 @@ func NewNode(t *testing.T, cfg *LitNodeConfig, cfg.LitTLSCertPath = filepath.Join(cfg.LitDir, "tls.cert") cfg.GenerateListeningPorts() + // Decide which DB backend to use. + switch *litDBBackend { + case terminal.DatabaseBackendSqlite: + cfg.DBBackend = terminal.DatabaseBackendSqlite + + case terminal.DatabaseBackendPostgres: + fixture := db.NewTestPgFixture( + t, db.DefaultPostgresFixtureLifetime, true, + ) + t.Cleanup(func() { + fixture.TearDown(t) + }) + + cfg.DBBackend = terminal.DatabaseBackendPostgres + cfg.PostgresConfig = fixture.GetConfig() + + default: + cfg.DBBackend = terminal.DatabaseBackendBbolt + } + + t.Logf("Using %v database backend", cfg.DBBackend) + // Generate a random UI password by reading 16 random bytes and base64 // encoding them. var randomBytes [16]byte diff --git a/make/testing_flags.mk b/make/testing_flags.mk index 0370bc29f..f3a261b6f 100644 --- a/make/testing_flags.mk +++ b/make/testing_flags.mk @@ -1,6 +1,5 @@ include make/compile_flags.mk -ITEST_FLAGS = TEST_FLAGS = DEV_TAGS = dev @@ -9,6 +8,11 @@ ifneq ($(icase),) ITEST_FLAGS += -test.run="TestLightningTerminal/$(icase)" endif +# Run itests with specified db backend. +ifneq ($(dbbackend),) +ITEST_FLAGS += -litdbbackend=$(dbbackend) +endif + # If a specific unit test case is being targeted, construct test.run filter. ifneq ($(case),) TEST_FLAGS += -test.run=$(case) diff --git a/terminal.go b/terminal.go index 590403d10..e297d0fe1 100644 --- a/terminal.go +++ b/terminal.go @@ -216,7 +216,7 @@ type LightningTerminal struct { middleware *mid.Manager middlewareStarted bool - accountsStore *accounts.BoltStore + accountsStore accounts.Store accountService *accounts.InterceptorService accountServiceStarted bool @@ -415,10 +415,14 @@ func (g *LightningTerminal) start(ctx context.Context) error { ) } - g.accountsStore, err = accounts.NewBoltStore( - filepath.Dir(g.cfg.MacaroonPath), accounts.DBFilename, - clock.NewDefaultClock(), - ) + networkDir := filepath.Join(g.cfg.LitDir, g.cfg.Network) + err = makeDirectories(networkDir) + if err != nil { + return fmt.Errorf("could not create network directory: %v", err) + } + + clock := clock.NewDefaultClock() + g.accountsStore, err = NewAccountStore(g.cfg, clock) if err != nil { return fmt.Errorf("error creating accounts store: %w", err) } @@ -445,10 +449,7 @@ func (g *LightningTerminal) start(ctx context.Context) error { g.ruleMgrs = rules.NewRuleManagerSet() // Create an instance of the local Terminal Connect session store DB. - networkDir := filepath.Join(g.cfg.LitDir, g.cfg.Network) - g.sessionDB, err = session.NewDB( - networkDir, session.DBFilename, clock.NewDefaultClock(), - ) + g.sessionDB, err = session.NewDB(networkDir, session.DBFilename, clock) if err != nil { return fmt.Errorf("error creating session DB: %v", err) }