lightningd: migrate events from bookkeeper at startup.

We take over the --bookkeeper-dir and --bookkeeper-db options, and
then if we can find the bookkeeper db we extract the records to
initialize our chain_moves and channel_moves tables.

Of course, bookkeeper now needs to not register those options.

When bookkeeper gets invoked the first time, it will reconstruct
everything from listchannelmoves and listcoinmoves.  It cannot
preserve manually-added descriptions, so we put those in the datastore
for it ready to go.

Note that the order of onchain_fee changes slightly from the original.
But this is fine.

Signed-off-by: Rusty Russell <[email protected]>

Author: rustyrussell

common/coin_mvt.c

@@ -155,7 +155,7 @@ const char *mvt_tag_str(enum mvt_tag tag)
 	return mvt_tags[tag];
 }
 
-static void tag_set(struct mvt_tags *tags, enum mvt_tag tag)
+void mvt_tag_set(struct mvt_tags *tags, enum mvt_tag tag)
 {
 	u64 bitnum = mvt_tag_in_db(tag);
 	assert(bitnum < NUM_MVT_TAGS);
@@ -394,10 +394,10 @@ struct chain_coin_mvt *new_coin_channel_open_proposed(const tal_t *ctx,
 
 	/* If we're the opener, add to the tag list */
 	if (is_opener)
-		tag_set(&tags, MVT_OPENER);
+		mvt_tag_set(&tags, MVT_OPENER);
 
 	if (is_leased)
-		tag_set(&tags, MVT_LEASED);
+		mvt_tag_set(&tags, MVT_LEASED);
 
 	mvt = new_chain_coin_mvt(ctx, channel, NULL, time_now().ts.tv_sec,
 				 NULL, out, NULL, 0,
@@ -423,10 +423,10 @@ struct chain_coin_mvt *new_coin_channel_open(const tal_t *ctx,
 
 	/* If we're the opener, add to the tag list */
 	if (is_opener)
-		tag_set(&tags, MVT_OPENER);
+		mvt_tag_set(&tags, MVT_OPENER);
 
 	if (is_leased)
-		tag_set(&tags, MVT_LEASED);
+		mvt_tag_set(&tags, MVT_LEASED);
 
 	mvt = new_chain_coin_mvt(ctx, channel, NULL, time_now().ts.tv_sec,
 				 NULL, out, NULL, blockheight,
@@ -684,10 +684,10 @@ struct mvt_tags mk_mvt_tags_(enum mvt_tag tag, ...)
 	va_list ap;
 	struct mvt_tags ret = { 0 };
 
-	tag_set(&ret, tag);
+	mvt_tag_set(&ret, tag);
 	va_start(ap, tag);
 	while ((tag = va_arg(ap, enum mvt_tag)) != 999)
-		tag_set(&ret, mvt_tag_in_db(tag));
+		mvt_tag_set(&ret, mvt_tag_in_db(tag));
 	va_end(ap);
 	return ret;
 }

common/coin_mvt.h

@@ -130,6 +130,9 @@ static inline struct mvt_tags tag_to_mvt_tags(enum mvt_tag tag)
 	return tags;
 }
 
+/* Add a tag */
+void mvt_tag_set(struct mvt_tags *tags, enum mvt_tag tag);
+
 /* Extract the primary tag */
 enum mvt_tag primary_mvt_tag(struct mvt_tags tags);
 

doc/lightningd-config.5.md

@@ -293,18 +293,6 @@ authenticate with username `user` and password `pass`, and then use the
 database `db_name`. The database must exist, but the schema will be managed
 automatically by `lightningd`.
 
-* **bookkeeper-dir**=*DIR* [plugin `bookkeeper`]
-
-  Directory to keep the accounts.sqlite3 database file in.
-Defaults to lightning-dir.
-
-* **bookkeeper-db**=*DSN* [plugin `bookkeeper`]
-
-  Identify the location of the bookkeeper data. This is a fully qualified data source
-name, including a scheme such as `sqlite3` or `postgres` followed by the
-connection parameters.
-Defaults to `sqlite3://accounts.sqlite3` in the `bookkeeper-dir`.
-
 * **encrypted-hsm**
 
  If set, you will be prompted to enter a password used to encrypt the `hsm_secret`.

lightningd/coin_mvts.c

@@ -258,35 +258,38 @@ void json_add_channel_mvt_fields(struct json_stream *stream,
 }
 
 static u64 coinmvt_index_inc(struct lightningd *ld,
+			     struct db *db,
 			     enum wait_subsystem subsys,
 			     const struct mvt_account_id *account,
 			     struct amount_msat credit,
 			     struct amount_msat debit,
 			     enum wait_index idx)
 {
-	return wait_index_increment(ld, subsys, idx,
+	return wait_index_increment(ld, db, subsys, idx,
 				    "account", account->channel ? fmt_channel_id(tmpctx, &account->channel->cid) : account->alt_account,
 				    "=credit_msat", tal_fmt(tmpctx, "%"PRIu64, credit.millisatoshis), /* Raw: JSON output */
 				    "=debit_msat", tal_fmt(tmpctx, "%"PRIu64, debit.millisatoshis), /* Raw: JSON output */
 				    NULL);
 }
 
 u64 chain_mvt_index_created(struct lightningd *ld,
+			    struct db *db,
 			    const struct mvt_account_id *account,
 			    struct amount_msat credit,
 			    struct amount_msat debit)
 {
-	return coinmvt_index_inc(ld, WAIT_SUBSYSTEM_CHAINMOVES,
+	return coinmvt_index_inc(ld, db, WAIT_SUBSYSTEM_CHAINMOVES,
 				 account, credit, debit,
 				 WAIT_INDEX_CREATED);
 }
 
 u64 channel_mvt_index_created(struct lightningd *ld,
+			      struct db *db,
 			      const struct mvt_account_id *account,
 			      struct amount_msat credit,
 			      struct amount_msat debit)
 {
-	return coinmvt_index_inc(ld, WAIT_SUBSYSTEM_CHANNELMOVES,
+	return coinmvt_index_inc(ld, db, WAIT_SUBSYSTEM_CHANNELMOVES,
 				 account, credit, debit,
 				 WAIT_INDEX_CREATED);
 }

lightningd/coin_mvts.h

@@ -4,6 +4,9 @@
 
 #include <common/coin_mvt.h>
 
+struct htlc_in;
+struct htlc_out;
+struct json_stream;
 struct lightningd;
 
 struct account_balance {
@@ -54,11 +57,13 @@ void json_add_channel_mvt_fields(struct json_stream *stream,
 
 /* For db code to get incremental ids */
 u64 chain_mvt_index_created(struct lightningd *ld,
+			    struct db *db,
 			    const struct mvt_account_id *account,
 			    struct amount_msat credit,
 			    struct amount_msat debit);
 
 u64 channel_mvt_index_created(struct lightningd *ld,
+			      struct db *db,
 			      const struct mvt_account_id *account,
 			      struct amount_msat credit,
 			      struct amount_msat debit);

lightningd/forwards.c

@@ -19,7 +19,8 @@ static u64 forward_index_inc(struct lightningd *ld,
 			     const struct short_channel_id *out_channel,
 			     enum wait_index idx)
 {
-	return wait_index_increment(ld, WAIT_SUBSYSTEM_FORWARD, idx,
+	return wait_index_increment(ld, ld->wallet->db,
+				    WAIT_SUBSYSTEM_FORWARD, idx,
 				    "status", forward_status_name(status),
 				    "in_channel", fmt_short_channel_id(tmpctx, in_channel),
 				    "=in_htlc_id", tal_fmt(tmpctx, "%"PRIu64, in_htlc_id),

lightningd/lightningd.c

@@ -271,6 +271,8 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	ld->try_reexec = false;
 	ld->recover_secret = NULL;
 	ld->db_upgrade_ok = NULL;
+	ld->old_bookkeeper_dir = NULL;
+	ld->old_bookkeeper_db = NULL;
 
 	/* --invoices-onchain-fallback */
 	ld->unified_invoices = false;

lightningd/lightningd.h

@@ -383,6 +383,10 @@ struct lightningd {
 
 	char *wallet_dsn;
 
+	/* For migration from old accounts.db */
+	char *old_bookkeeper_dir;
+	char *old_bookkeeper_db;
+
 	bool encrypted_hsm;
 	/* What (additional) messages the HSM accepts */
 	u32 *hsm_capabilities;

lightningd/options.c

@@ -1603,6 +1603,16 @@ static void register_opts(struct lightningd *ld)
 		       "Re-enable a long-deprecated API (which will be removed entirely next version!)");
 	opt_register_logging(ld);
 
+	/* Old bookkeeper migration flags. */
+	opt_register_early_arg("--bookkeeper-dir",
+			       opt_set_talstr, NULL,
+			       &ld->old_bookkeeper_dir,
+			       opt_hidden);
+	opt_register_early_arg("--bookkeeper-db",
+			       opt_set_talstr, NULL,
+			       &ld->old_bookkeeper_db,
+			       opt_hidden);
+
 	dev_register_opts(ld);
 }
 

lightningd/pay.c

@@ -2118,7 +2118,8 @@ static u64 sendpay_index_inc(struct lightningd *ld,
 			     enum payment_status status,
 			     enum wait_index idx)
 {
-	return wait_index_increment(ld, WAIT_SUBSYSTEM_SENDPAY, idx,
+	return wait_index_increment(ld, ld->wallet->db,
+				    WAIT_SUBSYSTEM_SENDPAY, idx,
 				    "status", payment_status_to_string(status),
 				    "=partid", tal_fmt(tmpctx, "%"PRIu64, partid),
 				    "=groupid", tal_fmt(tmpctx, "%"PRIu64, groupid),