feat: add an example of paginator with limit and offset

Signed-off-by: Vincenzo Palazzo <[email protected]>

Author: vincenzopalazzo

lightningd/jsonrpc.h

@@ -45,7 +45,7 @@ struct command {
 	/* Optional output field filter. */
 	struct json_filter *filter;
 	/* Option filtering option */
-	const struct jsonrpc_paginator *paginator;
+	struct jsonrpc_paginator *paginator;
 };
 
 /**

lightningd/peer_htlcs.c

@@ -2927,15 +2927,16 @@ void json_add_forwarding_object(struct json_stream *response,
 	json_object_end(response);
 }
 
-static void listforwardings_add_forwardings(struct json_stream *response,
-					    struct wallet *wallet,
+static void listforwardings_add_forwardings(struct command *cmd,
+					    struct json_stream *response,
 					    enum forward_status status,
 					    const struct short_channel_id *chan_in,
 					    const struct short_channel_id *chan_out)
 {
 	const struct forwarding *forwardings;
 
-	forwardings = wallet_forwarded_payments_get(wallet, tmpctx, status, chan_in, chan_out);
+	forwardings = wallet_forwarded_payments_get(cmd->ld->wallet, tmpctx, status,
+						    chan_in, chan_out, cmd->paginator);
 
 	json_array_start(response, "forwards");
 	for (size_t i=0; i<tal_count(forwardings); i++) {
@@ -2978,11 +2979,12 @@ static struct command_result *json_listforwards(struct command *cmd,
 			     FORWARD_ANY),
 		   p_opt("in_channel", param_short_channel_id, &chan_in),
 		   p_opt("out_channel", param_short_channel_id, &chan_out),
+		   p_paginator(&cmd->paginator),
 		   NULL))
 		return command_param_failed();
 
 	response = json_stream_success(cmd);
-	listforwardings_add_forwardings(response, cmd->ld->wallet, *status, chan_in, chan_out);
+	listforwardings_add_forwardings(cmd, response, *status, chan_in, chan_out);
 
 	return command_success(cmd, response);
 }

plugins/topology.c

@@ -411,32 +411,6 @@ static struct command_result *json_listchannels(struct command *cmd,
 	return send_outreq(cmd->plugin, req);
 }
 
-static bool node_in_paginator(struct command *cmd, struct node_id *node_id)
-{
-	struct jsonrpc_paginator *p;
-
-	p = cmd->paginator;
-	if (p) {
-		if (p->batch) {
-			const char *target_id;
-			const char *id;
-			size_t i;
-
-			target_id = node_id_to_hexstr(cmd, node_id);
-			/* FIXME: can we use the hash map to speed up the seach */
-			for (i = 0; i < tal_count(p->batch); i++) {
-				id = p->batch[i];
-				if (strcmp(id, target_id) == 0)
-					goto done;
-				plugin_log(cmd->plugin, LOG_DBG, "not match %s != %s", target_id, id);
-			}
-		}
-		return false;
-	}
-done:
-	return true;
-}
-
 static void json_add_node(struct command *cmd,
 			  struct json_stream *js,
 			  const struct gossmap *gossmap,
@@ -446,8 +420,6 @@ static void json_add_node(struct command *cmd,
 	u8 *nannounce;
 
 	gossmap_node_get_id(gossmap, n, &node_id);
-	if (!node_in_paginator(cmd, &node_id))
-		return;
 	json_object_start(js, NULL);
 	json_add_node_id(js, "nodeid", &node_id);
 	nannounce = gossmap_node_get_announce(tmpctx, gossmap, n);
@@ -532,6 +504,17 @@ static struct command_result *json_listnodes(struct command *cmd,
 		struct gossmap_node *n = gossmap_find_node(gossmap, id);
 		if (n)
 			json_add_node(cmd, js, gossmap, n);
+	} else if (cmd->paginator) {
+		const char **batch = cmd->paginator->batch;
+		for (size_t i = 0; i < tal_count(batch); i++) {
+			const char *idstr = batch[i];
+			id = tal(tmpctx, struct node_id);
+;			node_id_from_hexstr(idstr, sizeof(idstr), id);
+			struct gossmap_node *n = gossmap_find_node(gossmap, id);
+			if (n)
+				json_add_node(cmd, js, gossmap, n);
+			tal_free(id);
+		}
 	} else {
 		for (struct gossmap_node *n = gossmap_first_node(gossmap);
 		     n;

tests/test_paginator.py

@@ -2,8 +2,9 @@
 I think I'm getting to excited for this paginator API
 that I think this deserve a own python test file!
 """
+import unittest
 from fixtures import * # noqua: F401,F403
-
+from utils import COMPAT
 
 def test_listnodes_paginator(node_factory):
     """
@@ -13,9 +14,28 @@ def test_listnodes_paginator(node_factory):
     l1, l2, l3, _ = node_factory.line_graph(4, fundchannel=True, wait_for_announce=True)
     l1.rpc.jsonschemas = {}
 
-    nodes = l1.rpc.call("listnodes", { "batch": [l3.info["id"], l2.info["id"]] })
+    nodes = l1.rpc.call("listnodes", { "paginator": { "batch": [l3.info["id"], l2.info["id"]] } })
     nodes_nobatch = l1.rpc.listnodes()
     print(nodes)
     print(nodes_nobatch)
     assert len(nodes["nodes"]) == 2
     assert len(nodes_nobatch["nodes"]) == 4
+
+
+@unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Canned db used")
+@unittest.skipIf(not COMPAT, "needs COMPAT to convert obsolete db")
+@unittest.skipIf(TEST_NETWORK != 'regtest', "The DB migration is network specific due to the chain var.")
+def test_filter_listforwards_from_db(bitcoind, node_factory):
+    """This test is taken from the test_db and adapt to support the
+    paginator API. This allow to have a static way to assert over the
+    forwards without doing crazy things."""
+    bitcoind.generate_block(113)
+    l1 = node_factory.get_node(dbfile='v0.12.1-forward.sqlite3.xz',
+                               options={'database-upgrade': True})
+
+    assert l1.rpc.getinfo()['fees_collected_msat'] == 4
+    assert len(l1.rpc.listforwards()['forwards']) == 4
+    filter_forwards = l1.rpc.call("listforwards", { "paginator": { "limit": 2, "offset": 0 } })['forwards']
+    assert len(filter_forwards) == 2
+
+

wallet/test/run-wallet.c

@@ -1,6 +1,5 @@
 #include "config.h"
-  #include <lightningd/log.h>
-
+#include <lightningd/log.h>
 #include "test_utils.h"
 #include <ccan/tal/str/str.h>
 #include <db/common.h>
@@ -617,7 +616,11 @@ struct command_result *param_bool(struct command *cmd UNNEEDED, const char *name
 				  const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
 				  bool **b UNNEEDED)
 { fprintf(stderr, "param_bool called!\n"); abort(); }
-/* Generated stub for param_channel_id */
+/* Generated stub for param_bool */
+struct command_result *param_paginator(struct command *cmd UNNEEDED, const char *name UNNEEDED,
+				  const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
+				  struct jsonrpc_paginator **b UNNEEDED)
+{ fprintf(stderr, "param_paginator called!\n"); abort(); }/* Generated stub for param_channel_id */
 struct command_result *param_channel_id(struct command *cmd UNNEEDED,
 					const char *name UNNEEDED,
 					const char *buffer UNNEEDED,

wallet/wallet.c

@@ -4528,7 +4528,8 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 						       const tal_t *ctx,
 						       enum forward_status status,
 						       const struct short_channel_id *chan_in,
-						       const struct short_channel_id *chan_out)
+						       const struct short_channel_id *chan_out,
+	                                               const struct jsonrpc_paginator *paginator)
 {
 	struct forwarding *results = tal_arr(ctx, struct forwarding, 0);
 	size_t count = 0;
@@ -4537,24 +4538,46 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 	// placeholder for any parameter, the value doesn't matter because it's discarded by sql
 	const int any = -1;
 
-	stmt = db_prepare_v2(
-	    w->db,
-	    SQL("SELECT"
-		"  state"
-		", in_msatoshi"
-		", out_msatoshi"
-		", in_channel_scid"
-		", out_channel_scid"
-		", in_htlc_id"
-		", out_htlc_id"
-		", received_time"
-		", resolved_time"
-		", failcode "
-		", forward_style "
-		"FROM forwards "
-		"WHERE (1 = ? OR state = ?) AND "
-		"(1 = ? OR in_channel_scid = ?) AND "
-		"(1 = ? OR out_channel_scid = ?)"));
+	if (paginator) {
+		stmt = db_prepare_v2(
+			w->db,
+			SQL("SELECT"
+			    "  state"
+			    ", in_msatoshi"
+			    ", out_msatoshi"
+			    ", in_channel_scid"
+			    ", out_channel_scid"
+			    ", in_htlc_id"
+			    ", out_htlc_id"
+			    ", received_time"
+			    ", resolved_time"
+			    ", failcode "
+			    ", forward_style "
+			    "FROM forwards "
+			    "WHERE (1 = ? OR state = ?) AND "
+			    "(1 = ? OR in_channel_scid = ?) AND "
+			    "(1 = ? OR out_channel_scid = ?)"
+			    "LIMIT ? OFFSET ?"));
+	} else {
+		stmt = db_prepare_v2(
+			w->db,
+			SQL("SELECT"
+			    "  state"
+			    ", in_msatoshi"
+			    ", out_msatoshi"
+			    ", in_channel_scid"
+			    ", out_channel_scid"
+			    ", in_htlc_id"
+			    ", out_htlc_id"
+			    ", received_time"
+			    ", resolved_time"
+			    ", failcode "
+			    ", forward_style "
+			    "FROM forwards "
+			    "WHERE (1 = ? OR state = ?) AND "
+			    "(1 = ? OR in_channel_scid = ?) AND "
+			    "(1 = ? OR out_channel_scid = ?)"));
+	}
 
 	if (status == FORWARD_ANY) {
 		// any status
@@ -4586,6 +4609,12 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 		db_bind_int(stmt, 5, any);
 	}
 
+	if (paginator) {
+		assert(paginator->limit && paginator->offset);
+		db_bind_int(stmt, 6, *paginator->limit);
+		db_bind_int(stmt, 7, *paginator->offset);
+	}
+
 	db_query_prepared(stmt);
 
 	for (count=0; db_step(stmt); count++) {

wallet/wallet.h

@@ -6,6 +6,7 @@
 #include <common/onion_encode.h>
 #include <common/penalty_base.h>
 #include <common/utxo.h>
+#include <common/jsonrpc_paginator.h>
 #include <common/wallet.h>
 #include <lightningd/bitcoind.h>
 #include <lightningd/log.h>
@@ -1359,7 +1360,8 @@ const struct forwarding *wallet_forwarded_payments_get(struct wallet *w,
 						       const tal_t *ctx,
 						       enum forward_status state,
 						       const struct short_channel_id *chan_in,
-						       const struct short_channel_id *chan_out);
+						       const struct short_channel_id *chan_out,
+	                                               const struct jsonrpc_paginator *paginator);
 
 /**
  * Delete a particular forward entry