Towards better testing for routing, using a canned gossmap from my mainnet node.

devtools/gossmap-compress.c

@@ -84,11 +84,12 @@ static unsigned int verbose = 0;
 #define GC_HEADERLEN (sizeof(GC_HEADER))
 #define GOSSIP_STORE_VER ((0 << 5) | 14)
 
+/* Backwards, we want larger first */
 static int cmp_node_num_chans(struct gossmap_node *const *a,
 			      struct gossmap_node *const *b,
 			      void *unused)
 {
-	return (int)(*a)->num_chans - (int)(*b)->num_chans;
+	return (int)(*b)->num_chans - (int)(*a)->num_chans;
 }
 
 static void write_bigsize(gzFile outf, u64 val)
@@ -495,10 +496,44 @@ static char *opt_node(const char *optarg, const struct pubkey ***node_ids)
 	return NULL;
 }
 
+static const char *get_alias(const tal_t *ctx,
+			     const struct gossmap *gossmap,
+			     const struct gossmap_node *n)
+{
+	const u8 *ann = gossmap_node_get_announce(tmpctx, gossmap, n);
+	secp256k1_ecdsa_signature signature;
+	u8 *features;
+	u32 timestamp;
+	struct node_id node_id;
+	u8 rgb_color[3];
+	u8 alias[32];
+	u8 *addresses;
+	struct tlv_node_ann_tlvs *tlvs;
+
+	if (!fromwire_node_announcement(tmpctx, ann, &signature, &features, &timestamp,
+					&node_id, rgb_color, alias, &addresses,
+					&tlvs))
+		return "";
+	return tal_strndup(ctx, (const char *)alias, 32);
+}
+
+static void cupdate_bad(struct gossmap *map,
+			const struct short_channel_id_dir *scidd,
+			u16 cltv_expiry_delta,
+			u32 fee_base_msat,
+			u32 fee_proportional_millionths,
+			void *unused)
+{
+	warnx("Bad cupdate for %s, ignoring (delta=%u, fee=%u/%u)",
+	      fmt_short_channel_id_dir(tmpctx, scidd),
+	      cltv_expiry_delta, fee_base_msat, fee_proportional_millionths);
+}
+
 int main(int argc, char *argv[])
 {
 	int infd, outfd;
 	const struct pubkey **node_ids;
+	bool print_nodes = false;
 
 	common_setup(argv[0]);
 	setup_locale();
@@ -507,33 +542,36 @@ int main(int argc, char *argv[])
 	opt_register_noarg("--verbose|-v", opt_add_one, &verbose,
 			   "Print details (each additional gives more!).");
 	opt_register_arg("--node-map=num=<nodeid>", opt_node, NULL, &node_ids,
-			   "Map node num to <nodeid>");
+			   "Map node num to <nodeid> (decompress only)");
+	opt_register_noarg("--output-node-map", opt_set_bool, &print_nodes,
+			   "Output nodenumber:nodeid:alias for each node (compress only)");
 	opt_register_noarg("--help|-h", opt_usage_and_exit,
 			   "[decompress|compress] infile outfile\n"
 			   "Compress or decompress a gossmap file",
 			   "Print this message.");
 
 	opt_parse(&argc, argv, opt_log_stderr_exit);
 	if (argc != 4)
-		opt_usage_and_exit("Needs 4 arguments");
+		opt_usage_exit_fail("Needs 4 arguments");
 
 	infd = open(argv[2], O_RDONLY);
 	if (infd < 0)
-		opt_usage_and_exit(tal_fmt(tmpctx, "Cannot open %s for reading: %s",
-					   argv[2], strerror(errno)));
+		opt_usage_exit_fail(tal_fmt(tmpctx, "Cannot open %s for reading: %s",
+					    argv[2], strerror(errno)));
 	outfd = open(argv[3], O_WRONLY|O_CREAT|O_TRUNC, 0666);
 	if (outfd < 0)
-		opt_usage_and_exit(tal_fmt(tmpctx, "Cannot open %s for writing: %s",
-					   argv[3], strerror(errno)));
+		opt_usage_exit_fail(tal_fmt(tmpctx, "Cannot open %s for writing: %s",
+					    argv[3], strerror(errno)));
 
 	if (streq(argv[1], "compress")) {
 		struct gossmap_node **nodes, *n;
 		size_t *node_to_compr_idx;
 		size_t node_count, channel_count;
 		struct gossmap_chan **chans, *c;
+		bool *dirs;
 		gzFile outf = gzdopen(outfd, "wb9");
 
-		struct gossmap *gossmap = gossmap_load_fd(tmpctx, infd, NULL, NULL, NULL);
+		struct gossmap *gossmap = gossmap_load_fd(tmpctx, infd, cupdate_bad, NULL, NULL);
 		if (!gossmap)
 			opt_usage_and_exit("Cannot read gossmap");
 
@@ -552,8 +590,18 @@ int main(int argc, char *argv[])
 
 		/* Create map of gossmap index to compression index */
 		node_to_compr_idx = tal_arr(nodes, size_t, gossmap_max_node_idx(gossmap));
-		for (size_t i = 0; i < tal_count(nodes); i++)
+		for (size_t i = 0; i < tal_count(nodes); i++) {
 			node_to_compr_idx[gossmap_node_idx(gossmap, nodes[i])] = i;
+			if (print_nodes) {
+				struct node_id node_id;
+				gossmap_node_get_id(gossmap, nodes[i], &node_id);
+
+				printf("%zu:%s:%s\n",
+				       i,
+				       fmt_node_id(tmpctx, &node_id),
+				       get_alias(tmpctx, gossmap, nodes[i]));
+			}
+		}
 
 		if (gzwrite(outf, GC_HEADER, GC_HEADERLEN) == 0)
 			err(1, "Writing header");
@@ -568,27 +616,42 @@ int main(int argc, char *argv[])
 		if (verbose)
 			printf("%zu channels\n", channel_count);
 		chans = tal_arr(gossmap, struct gossmap_chan *, channel_count);
+		dirs = tal_arr(gossmap, bool, channel_count);
 
 		/*  * <CHANNEL_ENDS> := {channel_count} {start_nodeidx}*{channel_count} {end_nodeidx}*{channel_count} */
 		write_bigsize(outf, channel_count);
 		size_t chanidx = 0;
 		/* We iterate nodes to get to channels.  This gives us nicer ordering for compression */
-		for (size_t wanted_dir = 0; wanted_dir < 2; wanted_dir++) {
-			for (n = gossmap_first_node(gossmap); n; n = gossmap_next_node(gossmap, n)) {
-				for (size_t i = 0; i < n->num_chans; i++) {
-					int dir;
-					c = gossmap_nth_chan(gossmap, n, i, &dir);
-					if (dir != wanted_dir)
-						continue;
-
-					write_bigsize(outf,
-						      node_to_compr_idx[gossmap_node_idx(gossmap, n)]);
-					/* First time reflects channel index for reader */
-					if (wanted_dir == 0)
-						chans[chanidx++] = c;
-				}
+		for (size_t i = 0; i < tal_count(nodes); i++) {
+			n = nodes[i];
+			for (size_t j = 0; j < n->num_chans; j++) {
+				const struct gossmap_node *peer;
+				int dir;
+				c = gossmap_nth_chan(gossmap, n, j, &dir);
+
+				peer = gossmap_nth_node(gossmap, c, !dir);
+				/* Don't write if peer already wrote it! */
+				/* FIXME: What about self-channels? */
+				if (node_to_compr_idx[gossmap_node_idx(gossmap, peer)] < i)
+					continue;
+
+				write_bigsize(outf, node_to_compr_idx[gossmap_node_idx(gossmap, n)]);
+
+				assert(chanidx < channel_count);
+				dirs[chanidx] = dir;
+				chans[chanidx] = c;
+				chanidx++;
 			}
 		}
+		assert(chanidx == channel_count);
+
+		/* Now write out the other ends of the channels */
+		for (size_t i = 0; i < channel_count; i++) {
+			const struct gossmap_node *peer;
+
+			peer = gossmap_nth_node(gossmap, chans[i], !dirs[i]);
+			write_bigsize(outf, node_to_compr_idx[gossmap_node_idx(gossmap, peer)]);
+		}
 
 		/* <DISABLEDS> := <DISABLED>* {channel_count*2} */
 		/* <DISABLED> := {chanidx}*2+{direction} */

plugins/askrene/mcf.c

@@ -2,7 +2,6 @@
 #include <assert.h>
 #include <ccan/bitmap/bitmap.h>
 #include <ccan/list/list.h>
-#include <ccan/lqueue/lqueue.h>
 #include <ccan/tal/str/str.h>
 #include <ccan/tal/tal.h>
 #include <common/utils.h>
@@ -393,7 +392,7 @@ static s64 get_arc_flow(
 static u32 arc_tail(const struct linear_network *linear_network,
                     const struct arc arc)
 {
-	assert(arc.idx < tal_count(linear_network->arc_tail_node));
+	assert(arc.idx < linear_network->max_num_arcs);
 	return linear_network->arc_tail_node[ arc.idx ];
 }
 /* Helper function.
@@ -402,7 +401,7 @@ static u32 arc_head(const struct linear_network *linear_network,
                     const struct arc arc)
 {
 	const struct arc dual = arc_dual(arc);
-	assert(dual.idx < tal_count(linear_network->arc_tail_node));
+	assert(dual.idx < linear_network->max_num_arcs);
 	return linear_network->arc_tail_node[dual.idx];
 }
 
@@ -413,7 +412,7 @@ static struct arc node_adjacency_begin(
 		const struct linear_network * linear_network,
 		const u32 node)
 {
-	assert(node < tal_count(linear_network->node_adjacency_first_arc));
+	assert(node < linear_network->max_num_nodes);
 	return linear_network->node_adjacency_first_arc[node];
 }
 
@@ -430,7 +429,7 @@ static struct arc node_adjacency_next(
 		const struct linear_network *linear_network,
 		const struct arc arc)
 {
-	assert(arc.idx < tal_count(linear_network->node_adjacency_next_arc));
+	assert(arc.idx < linear_network->max_num_arcs);
 	return linear_network->node_adjacency_next_arc[arc.idx];
 }
 
@@ -541,16 +540,13 @@ static void linear_network_add_adjacenct_arc(
 		const u32 node_idx,
 		const struct arc arc)
 {
-	assert(arc.idx < tal_count(linear_network->arc_tail_node));
+	assert(arc.idx < linear_network->max_num_arcs);
 	linear_network->arc_tail_node[arc.idx] = node_idx;
 
-	assert(node_idx < tal_count(linear_network->node_adjacency_first_arc));
+	assert(node_idx < linear_network->max_num_nodes);
 	const struct arc first_arc = linear_network->node_adjacency_first_arc[node_idx];
 
-	assert(arc.idx < tal_count(linear_network->node_adjacency_next_arc));
 	linear_network->node_adjacency_next_arc[arc.idx]=first_arc;
-
-	assert(node_idx < tal_count(linear_network->node_adjacency_first_arc));
 	linear_network->node_adjacency_first_arc[node_idx]=arc;
 }
 
@@ -597,23 +593,23 @@ init_linear_network(const tal_t *ctx, const struct pay_parameters *params)
 	linear_network->max_num_nodes = max_num_nodes;
 
 	linear_network->arc_tail_node = tal_arr(linear_network,u32,max_num_arcs);
-	for(size_t i=0;i<tal_count(linear_network->arc_tail_node);++i)
+	for(size_t i=0;i<max_num_arcs;++i)
 		linear_network->arc_tail_node[i]=INVALID_INDEX;
 
 	linear_network->node_adjacency_next_arc = tal_arr(linear_network,struct arc,max_num_arcs);
-	for(size_t i=0;i<tal_count(linear_network->node_adjacency_next_arc);++i)
+	for(size_t i=0;i<max_num_arcs;++i)
 		linear_network->node_adjacency_next_arc[i].idx=INVALID_INDEX;
 
 	linear_network->node_adjacency_first_arc = tal_arr(linear_network,struct arc,max_num_nodes);
-	for(size_t i=0;i<tal_count(linear_network->node_adjacency_first_arc);++i)
+	for(size_t i=0;i<max_num_nodes;++i)
 		linear_network->node_adjacency_first_arc[i].idx=INVALID_INDEX;
 
 	linear_network->arc_prob_cost = tal_arr(linear_network,s64,max_num_arcs);
-	for(size_t i=0;i<tal_count(linear_network->arc_prob_cost);++i)
+	for(size_t i=0;i<max_num_arcs;++i)
 		linear_network->arc_prob_cost[i]=INFINITE;
 
 	linear_network->arc_fee_cost = tal_arr(linear_network,s64,max_num_arcs);
-	for(size_t i=0;i<tal_count(linear_network->arc_fee_cost);++i)
+	for(size_t i=0;i<max_num_arcs;++i)
 		linear_network->arc_fee_cost[i]=INFINITE;
 
 	linear_network->capacity = tal_arrz(linear_network,s64,max_num_arcs);
@@ -687,13 +683,6 @@ init_linear_network(const tal_t *ctx, const struct pay_parameters *params)
 	return linear_network;
 }
 
-/* Simple queue to traverse the network. */
-struct queue_data
-{
-	u32 idx;
-	struct lqueue_link ql;
-};
-
 // TODO(eduardo): unit test this
 /* Finds an admissible path from source to target, traversing arcs in the
  * residual network with capacity greater than 0.
@@ -705,25 +694,21 @@ find_admissible_path(const struct linear_network *linear_network,
 		     const u32 source, const u32 target, struct arc *prev)
 {
 	bool target_found = false;
+	/* Simple linear queue of node indexes */
+	u32 *queue = tal_arr(tmpctx, u32, linear_network->max_num_arcs);
+	size_t qstart, qend, prev_len = tal_count(prev);
 
-	for(size_t i=0;i<tal_count(prev);++i)
+	for(size_t i=0;i<prev_len;++i)
 		prev[i].idx=INVALID_INDEX;
 
 	// The graph is dense, and the farthest node is just a few hops away,
 	// hence let's BFS search.
-	LQUEUE(struct queue_data,ql) myqueue = LQUEUE_INIT;
-	struct queue_data *qdata;
-
-	qdata = tal(tmpctx, struct queue_data);
-	qdata->idx = source;
-	lqueue_enqueue(&myqueue,qdata);
+	queue[0] = source;
+	qstart = 0;
+	qend = 1;
 
-	while(!lqueue_empty(&myqueue))
-	{
-		qdata = lqueue_dequeue(&myqueue);
-		u32 cur = qdata->idx;
-
-		tal_free(qdata);
+	while (qstart < qend) {
+		u32 cur = queue[qstart++];
 
 		if(cur==target)
 		{
@@ -741,17 +726,15 @@ find_admissible_path(const struct linear_network *linear_network,
 
 			u32 next = arc_head(linear_network,arc);
 
-			assert(next < tal_count(prev));
+			assert(next < prev_len);
 
 			// if that node has been seen previously
 			if(prev[next].idx!=INVALID_INDEX)
 				continue;
 
 			prev[next] = arc;
-
-			qdata = tal(tmpctx, struct queue_data);
-			qdata->idx = next;
-			lqueue_enqueue(&myqueue,qdata);
+			assert(qend < linear_network->max_num_arcs);
+			queue[qend++] = next;
 		}
 	}
 	return target_found;