feat: implement a paginate in reverse flag

common/Makefile

@@ -109,6 +109,7 @@ COMMON_HEADERS_NOGEN := $(COMMON_SRC_NOGEN:.c=.h)	\
 	common/htlc.h					\
 	common/json_command.h				\
 	common/jsonrpc_errors.h				\
+       common/jsonrpc_paginator.h                      \
 	common/overflows.h
 
 COMMON_HEADERS_GEN := common/htlc_state_names_gen.h common/status_wiregen.h common/peer_status_wiregen.h common/scb_wiregen.h

common/json_param.c

@@ -1,4 +1,5 @@
 #include "config.h"
+#include <assert.h>
 #include <bitcoin/address.h>
 #include <bitcoin/base58.h>
 #include <bitcoin/feerate.h>
@@ -14,6 +15,10 @@
 #include <common/json_command.h>
 #include <common/json_param.h>
 #include <common/route.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
 
 struct param {
 	const char *name;
@@ -337,7 +342,7 @@ const char *param_subcommand(struct command *cmd, const char *buffer,
 }
 
 bool param(struct command *cmd, const char *buffer,
-	   const jsmntok_t tokens[], ...)
+	   const jsmntok_t *tokens, ...)
 {
 	struct param *params = tal_arr(tmpctx, struct param, 0);
 	const char *name;
@@ -353,6 +358,7 @@ bool param(struct command *cmd, const char *buffer,
 			allow_extra = true;
 			continue;
 		}
+
 		if (!param_add(&params, name, style, cbx, arg)) {
 			/* We really do ignore this return! */
 			struct command_result *ignore;
@@ -439,6 +445,37 @@ struct command_result *param_string(struct command *cmd, const char *name,
 	return NULL;
 }
 
+struct command_result *param_arr_str(struct command *cmd, const char *name,
+				   const char *buffer, const jsmntok_t *tok,
+				   const char ***arr)
+{
+	const jsmntok_t *curr;
+	size_t i;
+
+	if (tok->type != JSMN_ARRAY)
+		return command_fail_badparam(cmd, name, buffer, tok,
+					     "schould be an array of string");
+
+	*arr = tal_arr(cmd, const char *, 0);
+	json_for_each_arr(i, curr, tok) {
+		struct json_escape *esc;
+		const char *str;
+
+		if (curr->type != JSMN_STRING)
+			return command_fail_badparam(cmd, name, buffer, tok,
+						     "the item of the array should be a string");
+
+		esc = json_escape_string_(cmd, buffer + curr->start,
+			   curr->end - curr->start);
+		str = json_escape_unescape(cmd, esc);
+		tal_arr_expand(arr, str);
+	}
+
+	return NULL;
+}
+
+
+
 struct command_result *param_ignore(struct command *cmd, const char *name,
 				    const char *buffer, const jsmntok_t *tok,
 				    const void *unused)
@@ -1066,3 +1103,40 @@ struct command_result *param_pubkey(struct command *cmd, const char *name,
 				     "should be a compressed pubkey");
 }
 
+struct command_result *param_paginator(struct command *cmd, const char *name,
+				       const char *buffer, const jsmntok_t *tok,
+				       struct jsonrpc_paginator **paginator)
+{
+	const jsmntok_t *batch_tok, *offset_tok, *limit_tok, *reverse_tok;
+	u64 *limit, *offset;
+	bool *reverse;
+	const char **batch;
+
+	batch = NULL;
+	batch_tok = json_get_member(buffer, tok, "batch");
+	if (batch_tok)
+		json_to_strarr(cmd, buffer, batch_tok, &batch);
+
+	offset = tal(cmd, uint64_t);
+	offset_tok = json_get_member(buffer, tok, "offset");
+	if (offset_tok)
+		json_to_u64(buffer, offset_tok, offset);
+
+	limit = tal(cmd, uint64_t);
+	limit_tok = json_get_member(buffer, tok, "limit");
+	if (limit_tok)
+		json_to_u64(buffer, limit_tok, limit);
+
+	reverse = tal(cmd, bool);
+	reverse_tok = json_get_member(buffer, tok, "reverse");
+	if (reverse_tok)
+		json_to_bool(buffer, reverse_tok, reverse);
+
+	if (batch || (limit && offset)) {
+		*paginator = new_paginator(cmd, batch, limit, offset, reverse);
+		assert(paginator);
+		return NULL;
+	}
+	return command_fail_badparam(cmd, name, buffer, tok,
+				     "paginator request format in the wrong way!");
+}

common/json_param.h

@@ -2,9 +2,11 @@
 #ifndef LIGHTNING_COMMON_JSON_PARAM_H
 #define LIGHTNING_COMMON_JSON_PARAM_H
 #include "config.h"
+#include <assert.h>
 #include <ccan/short_types/short_types.h>
 #include <common/bolt11.h>
 #include <common/json_parse.h>
+#include <common/jsonrpc_paginator.h>
 #include <common/lease_rates.h>
 #include <common/node_id.h>
 #include <common/sphinx.h>
@@ -49,6 +51,10 @@ struct command_result;
 bool param(struct command *cmd, const char *buffer,
 	   const jsmntok_t params[], ...) LAST_ARG_NULL;
 
+bool param_partial_par(struct command *cmd, const char *buffer,
+		       const jsmntok_t tokens[], ...) LAST_ARG_NULL;
+
+
 /*
  * The callback signature.
  *
@@ -80,6 +86,11 @@ enum param_style {
 	PARAM_OPTIONAL_WITH_DEFAULT,
 };
 
+/** Check if this is a valid paginator input */
+struct command_result *param_paginator(struct command *cmd, const char *name,
+				       const char *buffer, const jsmntok_t *tok,
+	                               struct jsonrpc_paginator **paginator);
+
 /*
  * Add a required parameter.
  */
@@ -139,6 +150,18 @@ enum param_style {
 /* Special flag for 'check' which allows any parameters. */
 #define p_opt_any() "", PARAM_OPTIONAL, NULL, NULL
 
+#define p_paginator(arg)                                           \
+	            "paginator",				   \
+	            PARAM_OPTIONAL,				   \
+		    (param_cbx)(param_paginator),                  \
+	            ({ *arg = NULL;			           \
+		    (arg) + 0*sizeof((param_paginator)((struct command *)NULL, \
+		                  (const char *)NULL,              \
+				  (const char *)NULL,              \
+				  (const jsmntok_t *)NULL,         \
+				  (arg)) == (struct command_result *)NULL); })
+
+
 /* All the helper routines. */
 struct amount_msat;
 struct amount_sat;
@@ -181,6 +204,11 @@ struct command_result *param_string(struct command *cmd, const char *name,
 				    const char * buffer, const jsmntok_t *tok,
 				    const char **str);
 
+/* Extract an array of strings */
+struct command_result *param_arr_str(struct command *cmd, const char *name,
+				   const char *buffer, const jsmntok_t *tok,
+				   const char ***arr);
+
 /* Extract a label. It is either an escaped string or a number. */
 struct command_result *param_label(struct command *cmd, const char *name,
 				   const char * buffer, const jsmntok_t *tok,
@@ -340,5 +368,4 @@ struct command_result *param_lease_hex(struct command *cmd,
 struct command_result *param_pubkey(struct command *cmd, const char *name,
 				    const char *buffer, const jsmntok_t *tok,
 				    struct pubkey **pubkey);
-
 #endif /* LIGHTNING_COMMON_JSON_PARAM_H */

common/json_parse.c

@@ -711,3 +711,28 @@ json_tok_channel_id(const char *buffer, const jsmntok_t *tok,
 	return hex_decode(buffer + tok->start, tok->end - tok->start,
 			  cid, sizeof(*cid));
 }
+
+bool json_to_strarr(const tal_t *ctx, const char *buffer,
+		     const jsmntok_t *tok, const char ***arr)
+{
+	const jsmntok_t *curr;
+	size_t i;
+
+	if (tok->type != JSMN_ARRAY)
+		return false;
+
+	*arr = tal_arr(ctx, const char *, 0);
+	json_for_each_arr(i, curr, tok) {
+		struct json_escape *esc;
+		const char *str;
+
+		if (curr->type != JSMN_STRING)
+			return false;
+		esc = json_escape_string_(ctx, buffer + curr->start,
+			   curr->end - curr->start);
+		str = json_escape_unescape(ctx, esc);
+		tal_arr_expand(arr, str);
+	}
+
+	return NULL;
+}

common/json_parse.h

@@ -124,6 +124,9 @@ json_to_blinded_path(const tal_t *ctx, const char *buffer, const jsmntok_t *tok)
 bool json_tok_channel_id(const char *buffer, const jsmntok_t *tok,
 			 struct channel_id *cid);
 
+bool json_to_strarr(const tal_t *ctx, const char *buffer,
+		     const jsmntok_t *tok, const char ***arr);
+
 /* Guide is % for a token: each must be followed by JSON_SCAN().
  * Returns NULL on error (asserts() on bad guide). */
 const char *json_scan(const tal_t *ctx,

common/jsonrpc_paginator.h

@@ -0,0 +1,82 @@
+/**
+ * Welcome in this wanderful experience of writing
+ * a paginator for the core lightning JSON RPC 2.0 in C!
+ *
+ * I will try to keep the code as simple as possible,
+ * so if you had any dout on what I'm trying to do, blame me
+ * that I was not able to do my job.
+ *
+ * In short, the goal of this paginator is offer a struct
+ * that it is used to grep the paginator information by
+ * pre processin the JSON RPC request.
+ *
+ * So let immagine a normal listnode RPC call, where
+ * cli | rest | grpc -> json_listnodes -> json response {....}
+ *
+ * if the paginator will be enbaled on the listnodes, the
+ * call will be
+ *
+ * cli | rest | grpc -> json_paginator_listnodes -> json_listnodes + struct jsonrpc_paginator -> jon response { .... }
+ *
+ * Done, this should be all!
+ */
+#ifndef JSONRPC_PAGINATOR_H
+#define JSONRPC_PAGINATOR_H
+
+#include <assert.h>
+#include <ccan/tal/tal.h>
+#include <ccan/short_types/short_types.h>
+
+/**
+ * jsonrpc_paginator - core struct where all the paginator information
+ * are stored,
+ *
+ * A developer that want to extend the functionality of the
+ * paginator must not abuse of this struct to avoid to made the logic
+ * messy, thanks!
+ *
+ * @batch: array string to be able to query a list of things, useful also
+ * when you deal with reading stuff from file to avoid reaccess to the file
+ * to search another item.
+ *
+ * @offset: a position (u64) that determines the number of element (in SQL row)
+ * returned by request.
+ * @limit: a position (u64) that give the number of element in [0,..,offset - 1] to skip skip.
+ * @reverse: a boolean that reverse the order of the result.
+ */
+struct jsonrpc_paginator {
+	/** reall usefult for access to gossip map */
+	const char **batch;
+	/** query the database, for more complexy
+	 * query please use the sql plugins */
+	const u64 *offset;
+	const u64 *limit;
+	const bool *reverse;
+	/* FIXME: more smarter one? like sort_by = "json key"
+	 * but this required to have a mapping between json_keys and sql keys
+	 * maybe we had already somethings in the sql plugin? */
+};
+
+/**
+ * new_paginator - helper function to create a new paginator from a list of parameter.
+ * This is simple enought to be avoided, but write simple code inside the C macros
+ * is frustating, so let use this insteand.
+ *
+ * BTW: I love C Macros, really!
+ */
+static inline struct jsonrpc_paginator *
+new_paginator(const tal_t *ctx, const char **batch, const u64 *limit,
+	      const u64 *offset, const bool *reverse)
+{
+	struct jsonrpc_paginator *paginator = NULL;
+	if (batch || (limit && offset)) {
+		paginator = tal(ctx, struct jsonrpc_paginator);
+		paginator->batch = batch;
+		paginator->limit = limit;
+		paginator->offset = offset;
+		paginator->reverse = reverse;
+		return paginator;
+	}
+	return NULL;
+}
+#endif // JSONRPC_PAGINATOR_H

common/test/run-json_filter.c

@@ -72,6 +72,8 @@ const char *json_scan(const tal_t *ctx UNNEEDED,
 bool json_to_channel_id(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
 			struct channel_id *cid UNNEEDED)
 { fprintf(stderr, "json_to_channel_id called!\n"); abort(); }
+bool json_to_strarr(const tal_t *ctx UNNEEDED, const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED, const char ***arr UNNEEDED)
+{ fprintf(stderr, "json_to_strarr called"); abort();}
 /* Generated stub for json_to_millionths */
 bool json_to_millionths(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
 			u64 *millionths UNNEEDED)

common/test/run-json_remove.c

@@ -133,6 +133,9 @@ bool json_to_txid(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
 bool json_to_u16(const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED,
                  uint16_t *num UNNEEDED)
 { fprintf(stderr, "json_to_u16 called!\n"); abort(); }
+bool json_to_strarr(const tal_t *ctx UNNEEDED, const char *buffer UNNEEDED,
+		    const jsmntok_t *tok UNNEEDED, const char ***arr UNNEEDED)
+{ fprintf(stderr, "json_to_strarr called"); abort();}
 /* Generated stub for json_tok_bin_from_hex */
 u8 *json_tok_bin_from_hex(const tal_t *ctx UNNEEDED, const char *buffer UNNEEDED, const jsmntok_t *tok UNNEEDED)
 { fprintf(stderr, "json_tok_bin_from_hex called!\n"); abort(); }

lightningd/jsonrpc.h

@@ -4,6 +4,7 @@
 #include <ccan/list/list.h>
 #include <common/autodata.h>
 #include <common/json_stream.h>
+#include <common/jsonrpc_paginator.h>
 #include <common/status_levels.h>
 
 struct jsonrpc;
@@ -43,6 +44,8 @@ struct command {
 	struct json_stream *json_stream;
 	/* Optional output field filter. */
 	struct json_filter *filter;
+	/* Option filtering option */
+	struct jsonrpc_paginator *paginator;
 };
 
 /**
@@ -223,6 +226,7 @@ struct jsonrpc_notification *jsonrpc_notification_start(const tal_t *ctx, const
  */
 void jsonrpc_notification_end(struct jsonrpc_notification *n);
 
+
 /**
  * start a JSONRPC request; id_prefix is non-NULL if this was triggered by
  * another JSONRPC request.
@@ -273,4 +277,21 @@ void jsonrpc_request_end(struct jsonrpc_request *request);
 
 AUTODATA_TYPE(json_command, struct json_command);
 
+#define PAGINATOR(callback)						\
+	static struct command_result* callback##_paginator(struct command *cmd,         \
+	                                     const char *buffer,                        \
+	                                     const jsmntok_t *obj UNNEEDED,             \
+	                                     const jsmntok_t *params)                   \
+	{                                                                               \
+		const char **batch;						        \
+		u64 *limit, *offset;                                                    \
+                if (!param_partial_par(cmd, buffer, params,				\
+	             p_opt("batch", param_arr_str, &batch),                             \
+                     p_opt("limit", param_u64, &limit),                                 \
+		     p_opt("offset", param_u64, &offset),                               \
+		     NULL))                                                             \
+		     return command_param_failed();                                     \
+		cmd->paginator = new_paginator(cmd, batch, limit, offset);              \
+		return callback(cmd, buffer, obj, params);		                \
+        }
 #endif /* LIGHTNING_LIGHTNINGD_JSONRPC_H */