Merge pull request #773 from bhandras/cost-migraton-pagination

loop: paginate fetching payments when running the cost migration

Author: bhandras

cost_migration.go

@@ -14,7 +14,12 @@ import (
 )
 
 const (
+	// costMigrationID is the identifier for the cost migration.
 	costMigrationID = "cost_migration"
+
+	// paymentBatchSize is the maximum number of payments we'll fetch in
+	// one go.
+	paymentBatchSize = 1000
 )
 
 // CalculateLoopOutCost calculates the total cost of a loop out swap. It will
@@ -132,18 +137,30 @@ func MigrateLoopOutCosts(ctx context.Context, lnd lndclient.LndServices,
 		return err
 	}
 
-	// Next we fetch all payments from LND.
-	payments, err := lnd.Client.ListPayments(
-		ctx, lndclient.ListPaymentsRequest{},
-	)
-	if err != nil {
-		return err
-	}
-
 	// Gather payment fees to a map for easier lookup.
 	paymentFees := make(map[lntypes.Hash]lnwire.MilliSatoshi)
-	for _, payment := range payments.Payments {
-		paymentFees[payment.Hash] = payment.Fee
+	offset := uint64(0)
+
+	for {
+		payments, err := lnd.Client.ListPayments(
+			ctx, lndclient.ListPaymentsRequest{
+				Offset:      offset,
+				MaxPayments: paymentBatchSize,
+			},
+		)
+		if err != nil {
+			return err
+		}
+
+		if len(payments.Payments) == 0 {
+			break
+		}
+
+		for _, payment := range payments.Payments {
+			paymentFees[payment.Hash] = payment.Fee
+		}
+
+		offset = payments.LastIndexOffset + 1
 	}
 
 	// Now we'll calculate the cost for each swap and finally update the

loopd/daemon.go

@@ -412,6 +412,8 @@ func (d *Daemon) initialize(withMacaroonService bool) error {
 	// Run the costs migration.
 	err = loop.MigrateLoopOutCosts(d.mainCtx, d.lnd.LndServices, swapDb)
 	if err != nil {
+		log.Errorf("Cost migration failed: %v", err)
+
 		return err
 	}
 

test/lightning_client_mock.go

@@ -278,11 +278,24 @@ func (h *mockLightningClient) ListInvoices(_ context.Context,
 
 // ListPayments makes a paginated call to our list payments endpoint.
 func (h *mockLightningClient) ListPayments(_ context.Context,
-	_ lndclient.ListPaymentsRequest) (*lndclient.ListPaymentsResponse,
+	req lndclient.ListPaymentsRequest) (*lndclient.ListPaymentsResponse,
 	error) {
 
+	if req.Offset >= uint64(len(h.lnd.Payments)) {
+		return &lndclient.ListPaymentsResponse{}, nil
+	}
+
+	lastIndexOffset := req.Offset + req.MaxPayments
+	if lastIndexOffset > uint64(len(h.lnd.Payments)) {
+		lastIndexOffset = uint64(len(h.lnd.Payments))
+	}
+
+	result := h.lnd.Payments[req.Offset:lastIndexOffset]
+
 	return &lndclient.ListPaymentsResponse{
-		Payments: h.lnd.Payments,
+		Payments:         result,
+		FirstIndexOffset: req.Offset,
+		LastIndexOffset:  lastIndexOffset - 1,
 	}, nil
 }