bkpr first use after large migration issues: the key fixes

plugins/bkpr/bookkeeper.c

@@ -53,6 +53,10 @@ static struct refresh_info *use_rinfo(struct refresh_info *rinfo)
 	return rinfo;
 }
 
+/* Recursion */
+static struct command_result *limited_listchannelmoves(struct command *cmd,
+						       struct refresh_info *rinfo);
+
 static struct command_result *rinfo_one_done(struct command *cmd,
 					     struct refresh_info *rinfo)
 {
@@ -123,16 +127,37 @@ static struct command_result *listchannelmoves_done(struct command *cmd,
 				     &be_index, sizeof(be_index),
 				     "create-or-replace",
 				     datastore_done, NULL, use_rinfo(rinfo));
+
+	/* If there might be more, try asking for more */
+	if (moves->size != 0)
+		limited_listchannelmoves(cmd, rinfo);
+
 	return rinfo_one_done(cmd, rinfo);
 }
 
+/* We do 1000 at a time to avoid overwhelming lightningd */
+static struct command_result *limited_listchannelmoves(struct command *cmd,
+						       struct refresh_info *rinfo)
+{
+	struct bkpr *bkpr = bkpr_of(cmd->plugin);
+	struct out_req *req;
+
+	req = jsonrpc_request_start(cmd, "listchannelmoves",
+				    listchannelmoves_done,
+				    plugin_broken_cb,
+				    use_rinfo(rinfo));
+	json_add_string(req->js, "index", "created");
+	json_add_u64(req->js, "start", bkpr->channelmoves_index + 1);
+	json_add_u64(req->js, "limit", 1000);
+	return send_outreq(req);
+}
+
 static struct command_result *listchainmoves_done(struct command *cmd,
 						  const char *method,
 						  const char *buf,
 						  const jsmntok_t *result,
 						  struct refresh_info *rinfo)
 {
-	struct out_req *req;
 	const jsmntok_t *moves, *t;
 	size_t i;
 	struct bkpr *bkpr = bkpr_of(cmd->plugin);
@@ -148,13 +173,7 @@ static struct command_result *listchainmoves_done(struct command *cmd,
 				     "create-or-replace",
 				     datastore_done, NULL, use_rinfo(rinfo));
 
-	req = jsonrpc_request_start(cmd, "listchannelmoves",
-				    listchannelmoves_done,
-				    plugin_broken_cb,
-				    use_rinfo(rinfo));
-	json_add_string(req->js, "index", "created");
-	json_add_u64(req->js, "start", bkpr->channelmoves_index + 1);
-	send_outreq(req);
+	limited_listchannelmoves(cmd, rinfo);
 	return rinfo_one_done(cmd, rinfo);
 }
 

plugins/libplugin.c

@@ -290,17 +290,6 @@ static void ld_rpc_send(struct plugin *plugin, struct json_stream *stream)
 		io_wake(plugin->io_rpc_conn);
 }
 
-
-/* When cmd for request is gone, we use this as noop callback */
-static struct command_result *ignore_cb(struct command *command,
-					const char *method,
-					const char *buf,
-					const jsmntok_t *result,
-					void *arg)
-{
-	return &complete;
-}
-
 /* Ignore the result, and terminate the timer/aux/hook */
 struct command_result *ignore_and_complete(struct command *cmd,
 					   const char *method,
@@ -357,14 +346,6 @@ struct command_result *plugin_broken_cb(struct command *cmd,
 		   json_tok_full(buf, result));
 }
 
-static void disable_request_cb(struct command *cmd, struct out_req *out)
-{
-	out->errcb = NULL;
-	out->cb = ignore_cb;
-	/* Called because cmd got free'd */
-	out->cmd = NULL;
-}
-
 /* Prefix is usually a cmd->id */
 static const char *json_id(const tal_t *ctx, struct plugin *plugin,
 			   const char *method, const char *prefix)
@@ -424,9 +405,6 @@ jsonrpc_request_start_(struct command *cmd,
 	strmap_add(&cmd->plugin->out_reqs, out->id, out);
 	tal_add_destructor2(out, destroy_out_req, cmd->plugin);
 
-	/* If command goes away, don't call callbacks! */
-	tal_add_destructor2(out->cmd, disable_request_cb, out);
-
 	out->js = new_json_stream(NULL, cmd, NULL);
 	json_object_start(out->js, NULL);
 	json_add_string(out->js, "jsonrpc", "2.0");
@@ -1100,9 +1078,6 @@ static void handle_rpc_reply(struct plugin *plugin, const char *buf, const jsmnt
 		return;
 	}
 
-	/* Remove destructor if one existed */
-	tal_del_destructor2(out->cmd, disable_request_cb, out);
-
 	/* We want to free this if callback doesn't. */
 	tal_steal(tmpctx, out);
 

plugins/sql.c

@@ -197,6 +197,10 @@ static const struct index indices[] = {
 		"channelmoves",
 		{ "account_id", NULL },
 	},
+	{
+		"channelmoves",
+		{ "payment_hash", NULL },
+	},
 };
 
 static enum fieldtype find_fieldtype(const jsmntok_t *name)
@@ -1757,20 +1761,22 @@ static const char *fmt_indexes(const tal_t *ctx, const char *table)
 	for (size_t i = 0; i < ARRAY_SIZE(indices); i++) {
 		if (!streq(indices[i].tablename, table))
 			continue;
-		/* FIXME: Handle multiple indices! */
-		assert(!ret);
+		if (!ret)
+			ret = tal_fmt(ctx, " indexed by ");
+		else
+			tal_append_fmt(&ret, ", also indexed by ");
 		BUILD_ASSERT(ARRAY_SIZE(indices[i].fields) == 2);
 		if (indices[i].fields[1])
-			ret = tal_fmt(tmpctx, "`%s` and `%s`",
-				      indices[i].fields[0],
-				      indices[i].fields[1]);
+			tal_append_fmt(&ret, "`%s` and `%s`",
+				       indices[i].fields[0],
+				       indices[i].fields[1]);
 		else
-			ret = tal_fmt(tmpctx, "`%s`",
-				      indices[i].fields[0]);
+			tal_append_fmt(&ret, "`%s`",
+				       indices[i].fields[0]);
 	}
 	if (!ret)
 		return "";
-	return tal_fmt(ctx, " indexed by %s", ret);
+	return ret;
 }
 
 static const char *json_prefix(const tal_t *ctx,

tests/test_plugin.py

@@ -3795,7 +3795,7 @@ def test_sql(node_factory, bitcoind):
                         {'name': 'extra_tags',
                          'type': 'string'}]},
         'channelmoves': {
-            'indices': [['account_id']],
+            'indices': [['account_id'], ['payment_hash']],
             'columns': [{'name': 'created_index',
                          'type': 'u64'},
                         {'name': 'account_id',