Merge pull request #8052 from bhandras/sql-invoices

sqldb: `InvoiceDB` implementation

Author: Roasbeef

.github/workflows/main.yml

@@ -252,6 +252,10 @@ jobs:
             args: backend=bitcoind dbbackend=postgres
           - name: bitcoind-sqlite
             args: backend=bitcoind dbbackend=sqlite
+          - name: bitcoind-postgres-nativesql
+            args: backend=bitcoind dbbackend=postgres nativesql=true
+          - name: bitcoind-sqlite-nativesql
+            args: backend=bitcoind dbbackend=sqlite nativesql=true
           - name: neutrino
             args: backend=neutrino
     steps:

config_builder.go

@@ -3,6 +3,7 @@ package lnd
 import (
 	"bytes"
 	"context"
+	"database/sql"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -31,6 +32,7 @@ import (
 	"github.com/lightningnetwork/lnd/chainntnfs"
 	"github.com/lightningnetwork/lnd/chainreg"
 	"github.com/lightningnetwork/lnd/channeldb"
+	"github.com/lightningnetwork/lnd/clock"
 	"github.com/lightningnetwork/lnd/invoices"
 	"github.com/lightningnetwork/lnd/keychain"
 	"github.com/lightningnetwork/lnd/kvdb"
@@ -42,6 +44,7 @@ import (
 	"github.com/lightningnetwork/lnd/macaroons"
 	"github.com/lightningnetwork/lnd/rpcperms"
 	"github.com/lightningnetwork/lnd/signal"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/walletunlocker"
 	"github.com/lightningnetwork/lnd/watchtower"
 	"github.com/lightningnetwork/lnd/watchtower/wtclient"
@@ -869,6 +872,11 @@ type DatabaseInstances struct {
 	// WalletDB is the configuration for loading the wallet database using
 	// the btcwallet's loader.
 	WalletDB btcwallet.LoaderOption
+
+	// NativeSQLStore is a pointer to a native SQL store that can be used
+	// for native SQL queries for tables that already support it. This may
+	// be nil if the use-native-sql flag was not set.
+	NativeSQLStore *sqldb.BaseDB
 }
 
 // DefaultDatabaseBuilder is a type that builds the default database backends
@@ -923,10 +931,11 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 	// state DB point to the same local or remote DB and the same namespace
 	// within that DB.
 	dbs := &DatabaseInstances{
-		HeightHintDB: databaseBackends.HeightHintDB,
-		MacaroonDB:   databaseBackends.MacaroonDB,
-		DecayedLogDB: databaseBackends.DecayedLogDB,
-		WalletDB:     databaseBackends.WalletDB,
+		HeightHintDB:   databaseBackends.HeightHintDB,
+		MacaroonDB:     databaseBackends.MacaroonDB,
+		DecayedLogDB:   databaseBackends.DecayedLogDB,
+		WalletDB:       databaseBackends.WalletDB,
+		NativeSQLStore: databaseBackends.NativeSQLStore,
 	}
 	cleanUp := func() {
 		// We can just close the returned close functions directly. Even
@@ -1011,11 +1020,21 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 	// using the same struct (and DB backend) instance.
 	dbs.ChanStateDB = dbs.GraphDB
 
-	// For now the only InvoiceDB implementation is the *channeldb.DB.
-	//
-	// TODO(positiveblue): use a sql first implementation for this
-	// interface.
-	dbs.InvoiceDB = dbs.GraphDB
+	// Instantiate a native SQL invoice store if the flag is set.
+	if d.cfg.DB.UseNativeSQL {
+		executor := sqldb.NewTransactionExecutor(
+			dbs.NativeSQLStore,
+			func(tx *sql.Tx) sqldb.InvoiceQueries {
+				return dbs.NativeSQLStore.WithTx(tx)
+			},
+		)
+
+		dbs.InvoiceDB = sqldb.NewInvoiceStore(
+			executor, clock.NewDefaultClock(),
+		)
+	} else {
+		dbs.InvoiceDB = dbs.GraphDB
+	}
 
 	// Wrap the watchtower client DB and make sure we clean up.
 	if cfg.WtClient.Active {
@@ -1298,8 +1317,9 @@ func initNeutrinoBackend(ctx context.Context, cfg *Config, chainDir string,
 	)
 	switch {
 	case cfg.DB.Backend == kvdb.SqliteBackendName:
+		sqliteConfig := lncfg.GetSqliteConfigKVDB(cfg.DB.Sqlite)
 		db, err = kvdb.Open(
-			kvdb.SqliteBackendName, ctx, cfg.DB.Sqlite, dbPath,
+			kvdb.SqliteBackendName, ctx, sqliteConfig, dbPath,
 			lncfg.SqliteNeutrinoDBName, lncfg.NSNeutrinoDB,
 		)
 

docs/release-notes/release-notes-0.18.0.md

@@ -339,6 +339,10 @@
 * [Refactor UpdateInvoice](https://github.com/lightningnetwork/lnd/pull/8100) to
   make it simpler to adjust code to also support SQL InvoiceDB implementation.
 
+* [InvoiceDB implementation](https://github.com/lightningnetwork/lnd/pull/8052)
+  for SQL backends enabling new users to optionally use  an experimental native
+  SQL invoices database.
+
 ## Code Health
 
 * [Remove database pointers](https://github.com/lightningnetwork/lnd/pull/8117) 

invoices/invoiceregistry_test.go

@@ -3,6 +3,7 @@ package invoices_test
 import (
 	"context"
 	"crypto/rand"
+	"database/sql"
 	"fmt"
 	"math"
 	"testing"
@@ -18,6 +19,7 @@ import (
 	"github.com/lightningnetwork/lnd/lntypes"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/record"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/stretchr/testify/require"
 )
 
@@ -114,12 +116,60 @@ func TestInvoiceRegistry(t *testing.T) {
 		return db, testClock
 	}
 
+	// First create a shared Postgres instance so we don't spawn a new
+	// docker container for each test.
+	pgFixture := sqldb.NewTestPgFixture(
+		t, sqldb.DefaultPostgresFixtureLifetime,
+	)
+	t.Cleanup(func() {
+		pgFixture.TearDown(t)
+	})
+
+	makeSQLDB := func(t *testing.T, sqlite bool) (invpkg.InvoiceDB,
+		*clock.TestClock) {
+
+		var db *sqldb.BaseDB
+		if sqlite {
+			db = sqldb.NewTestSqliteDB(t).BaseDB
+		} else {
+			db = sqldb.NewTestPostgresDB(t, pgFixture).BaseDB
+		}
+
+		executor := sqldb.NewTransactionExecutor(
+			db, func(tx *sql.Tx) sqldb.InvoiceQueries {
+				return db.WithTx(tx)
+			},
+		)
+
+		testClock := clock.NewTestClock(testNow)
+
+		return sqldb.NewInvoiceStore(executor, testClock), testClock
+	}
+
 	for _, test := range testList {
 		test := test
 
-		t.Run(test.name, func(t *testing.T) {
+		t.Run(test.name+"_KV", func(t *testing.T) {
 			test.test(t, makeKeyValueDB)
 		})
+
+		t.Run(test.name+"_SQLite", func(t *testing.T) {
+			test.test(t,
+				func(t *testing.T) (
+					invpkg.InvoiceDB, *clock.TestClock) {
+
+					return makeSQLDB(t, true)
+				})
+		})
+
+		t.Run(test.name+"_Postgres", func(t *testing.T) {
+			test.test(t,
+				func(t *testing.T) (
+					invpkg.InvoiceDB, *clock.TestClock) {
+
+					return makeSQLDB(t, false)
+				})
+		})
 	}
 }