itest: extend liquidity edge cases for rfq htlc tracking

Author: GeorgeTsagk

itest/assets_test.go

@@ -714,6 +714,9 @@ func sendAssetKeySendPayment(t *testing.T, src, dst *HarnessNode, amt uint64,
 
 	result, err := getAssetPaymentResult(stream, false)
 	require.NoError(t, err)
+	if result.Status == lnrpc.Payment_FAILED {
+		t.Logf("Failure reason: %v", result.FailureReason)
+	}
 	require.Equal(t, expectedStatus, result.Status)
 
 	expectedReason := failReason.UnwrapOr(

itest/litd_custom_channels_test.go

@@ -1,6 +1,7 @@
 package itest
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"math"
@@ -18,12 +19,14 @@ import (
 	"github.com/lightninglabs/taproot-assets/tapchannel"
 	"github.com/lightninglabs/taproot-assets/taprpc"
 	"github.com/lightninglabs/taproot-assets/taprpc/mintrpc"
+	"github.com/lightninglabs/taproot-assets/taprpc/rfqrpc"
 	tchrpc "github.com/lightninglabs/taproot-assets/taprpc/tapchannelrpc"
 	"github.com/lightninglabs/taproot-assets/taprpc/universerpc"
 	"github.com/lightninglabs/taproot-assets/tapscript"
 	"github.com/lightningnetwork/lnd/fn"
 	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/invoicesrpc"
+	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
 	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
 	"github.com/lightningnetwork/lnd/lntest"
 	"github.com/lightningnetwork/lnd/lntest/port"
@@ -47,7 +50,7 @@ var (
 
 	shortTimeout = time.Second * 5
 
-	defaultPaymentStatus = fn.None[lnrpc.Payment_PaymentStatus]()
+	defaultPaymentStatusOpt = fn.None[lnrpc.Payment_PaymentStatus]()
 )
 
 var (
@@ -227,7 +230,7 @@ func testCustomChannelsLarge(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, false,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -242,7 +245,7 @@ func testCustomChannelsLarge(_ context.Context, net *NetworkHarness,
 
 	payInvoiceWithAssets(
 		t.t, fabia, erin, invoiceResp2.PaymentRequest, assetID, false,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice 2")
 
@@ -253,7 +256,7 @@ func testCustomChannelsLarge(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp3.PaymentRequest, assetID, false,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice 3")
 
@@ -445,7 +448,7 @@ func testCustomChannels(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, dave, charlie, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice back")
 
@@ -510,7 +513,7 @@ func testCustomChannels(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -555,7 +558,7 @@ func testCustomChannels(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -592,7 +595,7 @@ func testCustomChannels(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -613,7 +616,7 @@ func testCustomChannels(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after asset-to-asset")
 
@@ -949,7 +952,7 @@ func testCustomChannelsGroupedAsset(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -987,7 +990,7 @@ func testCustomChannelsGroupedAsset(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -1024,7 +1027,7 @@ func testCustomChannelsGroupedAsset(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")
 
@@ -1045,7 +1048,7 @@ func testCustomChannelsGroupedAsset(_ context.Context, net *NetworkHarness,
 	)
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, true,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after asset-to-asset")
 
@@ -1805,7 +1808,7 @@ func testCustomChannelsLiquidityEdgeCases(_ context.Context,
 			SatPerVByte: 5,
 		},
 	)
-	defer closeChannelAndAssert(t, net, dave, channelOp, false)
+	defer closeChannelAndAssert(t, net, dave, channelOp, true)
 
 	// This is the only public channel, we need everyone to be aware of it.
 	assertChannelKnown(t.t, charlie, channelOp)
@@ -1958,7 +1961,7 @@ func testCustomChannelsLiquidityEdgeCases(_ context.Context,
 
 	payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, false,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 
 	logBalance(t.t, nodes, assetID, "after big asset payment (asset "+
@@ -2004,7 +2007,7 @@ func testCustomChannelsLiquidityEdgeCases(_ context.Context,
 
 	payInvoiceWithAssets(
 		t.t, yara, dave, invoiceResp.PaymentRequest, assetID, false,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 
 	logBalance(t.t, nodes, assetID, "after big asset payment (asset "+
@@ -2020,10 +2023,11 @@ func testCustomChannelsLiquidityEdgeCases(_ context.Context,
 	logBalance(t.t, nodes, assetID, "after small payment (asset "+
 		"invoice, <354sats)")
 
-	// Edge case: Now Charlie creates an asset invoice to be paid for by
+	// Edge case: Now Dave creates an asset invoice to be paid for by
 	// Yara with satoshi. For the last hop we try to settle the invoice in
-	// satoshi, where we will check whether Charlie's strict forwarding
-	// works as expected.
+	// satoshi, where we will check whether Dave's strict forwarding works
+	// as expected. Charlie is only used as a dummy RFQ peer in this case,
+	// Yara totally ignored the RFQ hint and pays agnostically with sats.
 	invoiceResp = createAssetInvoice(
 		t.t, charlie, dave, 1, assetID,
 	)
@@ -2046,6 +2050,166 @@ func testCustomChannelsLiquidityEdgeCases(_ context.Context,
 
 	logBalance(t.t, nodes, assetID, "after failed payment (asset "+
 		"invoice, strict forwarding)")
+
+	// Edge case: Fabia gets all the asset liquidity on her side. Then
+	// generates an asset invoice to be paid for with assets by Charlie.
+	// Charlie will unleash multiple shards towards Fabia, exhausting the
+	// liquidity in the Erin<->Fabia channel. Dave should be able to detect
+	// those failures and account for those cancelled HTLCs in the RFQ
+	// tracking mechanism. Eventually we will slosh some funds from Fabia
+	// back to Erin in order for Erin to be able to forward the rest of the
+	// HTLCs, leading to the release of the preimage.
+
+	// Erin starts by sending half of the assets to Fabia's side. He also
+	// sends some raw sats to be used by Fabia later for the sloshing.
+	sendAssetKeySendPayment(
+		t.t, erin, fabia, 150_000, assetID,
+		fn.None[int64](), lnrpc.Payment_SUCCEEDED,
+		fn.None[lnrpc.PaymentFailureReason](),
+	)
+	sendKeySendPayment(t.t, erin, fabia, 20_000)
+
+	logBalance(t.t, nodes, assetID, "balance after 1st slosh")
+
+	// Fabia creates an asset invoice of 100k assets. There is currently
+	// not enough asset liquidity on Erin's side to forward HTLCs to satisfy
+	// this invoice.
+	invoiceResp = createAssetInvoice(t.t, erin, fabia, 100_000, assetID)
+
+	// We set the waiting period for the slosh payment to occur. This is set
+	// to half of the default payment timeout, as we want it to occur half
+	// way through the in-flight payment.
+	sloshWait := PaymentTimeout / 3
+
+	go func() {
+		// After a small delay (less than the payment timeout) Fabia
+		// sloshes back the asset liquidity to Erin. This should allow
+		// the payment by Charlie to eventually complete.
+		time.Sleep(sloshWait)
+		sendAssetKeySendPayment(
+			t.t, fabia, erin, 75_000, assetID,
+			fn.None[int64](), lnrpc.Payment_SUCCEEDED,
+			fn.None[lnrpc.PaymentFailureReason](),
+		)
+	}()
+
+	// To avoid goroutine uncertainty, we wait as much as the above routine
+	// minus a small delta. This is enough for us to be sure that the
+	// payment will complete due to the slosh.
+	timeoutChan = time.After(sloshWait - time.Millisecond*250)
+	done = make(chan bool, 1)
+
+	go func() {
+		// Now Charlie pays the invoice with assets. What happens on the
+		// Charlie-Dave channel doesn't matter. This payment will block
+		// until the previous slosh payment completes. That slosh will
+		// allow for the rest of the HTLCs to be forwarded.
+		payInvoiceWithAssets(
+			t.t, charlie, dave, invoiceResp.PaymentRequest, assetID,
+			true, defaultPaymentStatusOpt,
+		)
+
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		// If the payment completes before the slosh payment occurs then
+		// something went wrong, this is not the expected test case
+		// behavior.
+		t.Fatalf("payment completed before expected block period")
+	case <-timeoutChan:
+		// The expected time delay for the payment to complete has been
+		// passed, so now we wait for the payment to complete.
+		<-done
+	}
+
+	logBalance(t.t, nodes, assetID, "after htlc track cancel")
+
+	// Edge case: Charlie negotiates a quote with Dave which has a low max
+	// amount (~170k sats). Then Charlie creates an invoice with a total
+	// amount slightly larger than the max allowed in the quote (200k sats).
+	// Erin will try to pay that invoice with sats, in shards of max size
+	// 80k sats. Dave will eventually stop forwarding HTLCs as the RFQ HTLC
+	// tracking mechanism should stop them from being forwarded, as they
+	// violate the maximum allowed amount of the quote.
+
+	// Charlie starts by negotiating the quote.
+	res, err := charlieTap.RfqClient.AddAssetBuyOrder(
+		ctxb, &rfqrpc.AddAssetBuyOrderRequest{
+			AssetSpecifier: &rfqrpc.AssetSpecifier{
+				Id: &rfqrpc.AssetSpecifier_AssetId{
+					AssetId: assetID,
+				},
+			},
+			AssetMaxAmt:    10_000,
+			Expiry:         uint64(time.Now().Add(time.Hour).Unix()),
+			PeerPubKey:     dave.PubKey[:],
+			TimeoutSeconds: 10,
+		},
+	)
+	require.NoError(t.t, err)
+
+	quote, ok := res.Response.(*rfqrpc.AddAssetBuyOrderResponse_AcceptedQuote)
+	require.True(t.t, ok)
+
+	// We now manually add the invoice in order to inject the above,
+	// manually generated, quote.
+	iResp, err := charlie.AddInvoice(ctxb, &lnrpc.Invoice{
+		Memo:       "",
+		Value:      200_000,
+		RPreimage:  bytes.Repeat([]byte{11}, 32),
+		CltvExpiry: 60,
+		RouteHints: []*lnrpc.RouteHint{
+			&lnrpc.RouteHint{
+				HopHints: []*lnrpc.HopHint{
+					&lnrpc.HopHint{
+						NodeId: dave.PubKeyStr,
+						ChanId: quote.AcceptedQuote.Scid,
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t.t, err)
+
+	// Now Erin tries to pay the invoice. Since the multipart payment will
+	// have some of its shards failing the pathfinding logic will keep going
+	// and we won't see a payment failure but a timeout. If a final outcome
+	// is not produced within a reasonable amount of time, we assume the
+	// payment is still trying to find a route, therefore the HTLC rejection
+	// works.
+	timeoutChan = time.After(PaymentTimeout / 2)
+	done = make(chan bool, 1)
+
+	ctxc, cancel := context.WithCancel(context.Background())
+
+	//nolint:lll
+	go func() {
+		// payInvoiceWithSatoshi(t.t, erin, iResp, lnrpc.Payment_FAILED)
+		sendReq := &routerrpc.SendPaymentRequest{
+			PaymentRequest:   iResp.PaymentRequest,
+			TimeoutSeconds:   int32(PaymentTimeout.Seconds()),
+			MaxShardSizeMsat: 80_000_000,
+			FeeLimitMsat:     1_000_000,
+		}
+		stream, err := erin.RouterClient.SendPaymentV2(ctxc, sendReq)
+		if err == nil {
+			_, _ = getPaymentResult(stream)
+		}
+
+		done <- true
+	}()
+
+	select {
+	case <-done:
+		t.Fatalf("Payment should not produce a final outcome")
+
+	case <-timeoutChan:
+		cancel()
+	}
+
+	logBalance(t.t, nodes, assetID, "after small manual rfq")
 }
 
 // testCustomChannelsBalanceConsistency is a test that test the balance of nodes
@@ -2563,7 +2727,7 @@ func testCustomChannelsOraclePricing(_ context.Context,
 
 	numUnits, rate := payInvoiceWithAssets(
 		t.t, charlie, dave, invoiceResp.PaymentRequest, assetID, false,
-		defaultPaymentStatus,
+		defaultPaymentStatusOpt,
 	)
 	logBalance(t.t, nodes, assetID, "after invoice")