db: Backfill missing HTLC IDs in the forwards table

We have a primary key that is spanning the `in_channel_id` and the
`in_htcl_id`. The latter gets set to NULL when the HTLC and channel
gets deleted, so we coalesce with a random large number that is
unlikely to collide for the primary key.

Author: cdecker

devtools/sql-rewrite.py

@@ -75,6 +75,7 @@ def rewrite_single(self, q):
 
         typemapping = {
             r'BLOB': 'BYTEA',
+            r'_ROWID_': '(((ctid::text::point)[0]::bigint << 32) | (ctid::text::point)[1]::bigint)',  # Yeah, I know...
             r'CURRENT_TIMESTAMP\(\)': "EXTRACT(epoch FROM now())",
         }
 

tests/test_db.py

@@ -508,6 +508,9 @@ def test_db_forward_migrate(bitcoind, node_factory):
     assert l1.rpc.getinfo()['fees_collected_msat'] == 4
     assert len(l1.rpc.listforwards()['forwards']) == 4
 
+    # The two null in_htlc_id are replaced with bogus entries!
+    assert sum([f['in_htlc_id'] > 0xFFFFFFFFFFFF for f in l1.rpc.listforwards()['forwards']]) == 2
+
     # Make sure autoclean can handle these!
     l1.stop()
     l1.daemon.opts['autoclean-succeededforwards-age'] = 2

wallet/db.c

@@ -912,7 +912,10 @@ static struct migration dbmigrations[] = {
 	 ", PRIMARY KEY(in_channel_scid, in_htlc_id))"), NULL},
     {SQL("INSERT INTO forwards SELECT"
 	 " in_channel_scid"
-	 ", (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.in_htlc_id)"
+	 ", COALESCE("
+	 "    (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.in_htlc_id),"
+	 "    -_ROWID_"
+	 "  )"
 	 ", out_channel_scid"
 	 ", (SELECT channel_htlc_id FROM channel_htlcs WHERE id = forwarded_payments.out_htlc_id)"
 	 ", in_msatoshi"