sqldb: move native SQL payments migrations into mainline

cmd/commands/cmd_payments.go

@@ -1497,6 +1497,11 @@ var listPaymentsCommand = cli.Command{
 				"payments with creation date less than or " +
 				"equal to it",
 		},
+		cli.BoolFlag{
+			Name: "omit_hops",
+			Usage: "if set, omit hop-level route data to " +
+				"reduce query cost and response size",
+		},
 	},
 	Action: actionDecorator(listPayments),
 }
@@ -1514,6 +1519,7 @@ func listPayments(ctx *cli.Context) error {
 		CountTotalPayments: ctx.Bool("count_total_payments"),
 		CreationDateStart:  ctx.Uint64("creation_date_start"),
 		CreationDateEnd:    ctx.Uint64("creation_date_end"),
+		OmitHops:           ctx.Bool("omit_hops"),
 	}
 
 	payments, err := client.ListPayments(ctxc, req)

config_builder.go

@@ -1134,6 +1134,16 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 
 				// Set the invoice bucket tombstone to indicate
 				// that the migration has been completed.
+				//
+				// TODO(ziggie): The tombstone is currently
+				// set inside the SQL transaction callback,
+				// which is fragile: if the SQL transaction
+				// is retried (e.g. on a serialization
+				// error), the KV tombstone is written before
+				// the SQL commit is confirmed. Move this to
+				// run after ApplyAllMigrations returns so
+				// the tombstone is only set once the
+				// migration is durably committed.
 				d.logger.Debugf("Setting invoice bucket " +
 					"tombstone")
 
@@ -1173,15 +1183,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 						"payments to SQL: %w", err)
 				}
 
-				// Set the payments bucket tombstone to
-				// indicate that the migration has been
-				// completed.
-				d.logger.Debugf("Setting payments bucket " +
-					"tombstone")
-
-				return paymentsdb.SetPaymentsBucketTombstone(
-					dbs.ChanStateDB.Backend,
-				)
+				return nil
 			}
 
 			// Make sure we attach the custom migration function to
@@ -1260,6 +1262,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 			graphExecutor, graphDBOptions...,
 		)
 		if err != nil {
+			cleanUp()
 			err = fmt.Errorf("unable to get graph store: %w", err)
 			d.logger.Error(err)
 
@@ -1275,6 +1278,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 			baseDB, dbs.ChanStateDB.Backend,
 		)
 		if err != nil {
+			cleanUp()
 			err = fmt.Errorf("unable to get payments store: %w",
 				err)
 
@@ -1286,8 +1290,12 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 		// Check if the invoice bucket tombstone is set. If it is, we
 		// need to return and ask the user switch back to using the
 		// native SQL store.
+		//
+		// NOTE: The invoice bucket tombstone acts as the system-wide
+		// guard against switching back to KV mode.
 		ripInvoices, err := dbs.ChanStateDB.GetInvoiceBucketTombstone()
 		if err != nil {
+			cleanUp()
 			err = fmt.Errorf("unable to check invoice bucket "+
 				"tombstone: %w", err)
 			d.logger.Error(err)
@@ -1302,33 +1310,14 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 			return nil, nil, err
 		}
 
-		// Check if the payments bucket tombstone is set. If it is, we
-		// need to return and ask the user switch back to using the
-		// native SQL store.
-		ripPayments, err := paymentsdb.GetPaymentsBucketTombstone(
-			dbs.ChanStateDB.Backend,
-		)
-		if err != nil {
-			err = fmt.Errorf("unable to check payments bucket "+
-				"tombstone: %w", err)
-			d.logger.Error(err)
-
-			return nil, nil, err
-		}
-		if ripPayments {
-			err = fmt.Errorf("payments bucket tombstoned, please " +
-				"switch back to native SQL")
-			d.logger.Error(err)
-
-			return nil, nil, err
-		}
-
 		dbs.InvoiceDB = dbs.ChanStateDB
 
 		graphStore, err = graphdb.NewKVStore(
 			databaseBackends.GraphDB, graphDBOptions...,
 		)
 		if err != nil {
+			cleanUp()
+
 			return nil, nil, err
 		}
 

docs/release-notes/release-notes-0.21.0.md

@@ -158,6 +158,17 @@
 
 ## Deprecations
 
+### ⚠️ **Warning:** Deprecated fields in `lnrpc.Hop` will be removed in release version **0.22**
+
+  The following deprecated fields in the [`lnrpc.Hop`](https://lightning.engineering/api-docs/api/lnd/lightning/send-to-route-sync/#lnrpchop)
+  message will be removed:
+
+  | Field | Deprecated Since | Replacement |
+  |-------|------------------|-------------|
+  | `chan_capacity` | 0.7.1 | None |
+  | `amt_to_forward` | 0.7.1 | `amt_to_forward_msat` |
+  | `fee` | 0.7.1 | `fee_msat` |
+
 ### ⚠️ **Warning:** The deprecated fee rate option `--sat_per_byte` will be removed in release version **0.22**
 
   The deprecated `--sat_per_byte` option will be fully removed. This flag was
@@ -228,6 +239,12 @@
     migration](https://github.com/lightningnetwork/lnd/pull/10485) with
     comprehensive tests. The migration is currently dev-only, compiled behind
     the `test_db_postgres`, `test_db_sqlite`, or `test_native_sql` build tags.
+  * Various [SQL payment store
+    improvements](https://github.com/lightningnetwork/lnd/pull/10535):
+    optimize schema indexes, improve query performance for payment filtering
+    and failed attempt cleanup, fix cross-database timestamp handling, add
+    `omit_hops` option to `ListPayments` to reduce response size, and increase
+    the default SQLite cache size.
 
 
 ## Code Health