Merge pull request #1089 from lightninglabs/fix-migration-14

guggero · web-flow · commit 2110839696cb · 2024-08-15T12:08:11.000-06:00
tapdb: fix multiple issues around migration file handling
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -81,15 +81,15 @@ jobs:
       - name: Run test vector creation check
         run: make test-vector-check
 
-  migration-version-check:
+  migration-check:
     name: migration version check
     runs-on: ubuntu-latest
     steps:
       - name: git checkout
         uses: actions/checkout@v3
 
-      - name: Run migration version check
-        run: make migration-version-check
+      - name: Run migration check
+        run: make migration-check
 
   ########################
   # Compilation check.
diff --git a/Makefile b/Makefile
@@ -331,7 +331,10 @@ test-vector-check: gen-deterministic-test-vectors
 	@$(call print, "Checking deterministic test vectors.")
 	if test -n "$$(git status | grep -e ".json")"; then echo "Test vectors not updated"; git status; git diff; exit 1; fi
 
-migration-version-check:
+migration-check:
+	@$(call print, "Checking migration numbering.")
+	./scripts/check-migration-numbering.sh
+
 	@$(call print, "Checking migration version.")
 	./scripts/check-migration-latest-version.sh
 
diff --git a/scripts/check-migration-numbering.sh b/scripts/check-migration-numbering.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# The directory containing migration files.
+migrations_path="tapdb/sqlc/migrations"
+
+# Check if the directory exists
+if [ ! -d "$migrations_path" ]; then
+  echo "Directory $migrations_path does not exist."
+  exit 1
+fi
+
+# Get all unique prefixes (e.g., 000001, always 6 digits) from .up.sql files.
+prefixes=($(ls "$migrations_path"/*.up.sql 2>/dev/null | \
+    sed -E 's/.*\/([0-9]{6})_.*\.up\.sql/\1/' | sort))
+
+# Check if no prefixes were found.
+if [ ${#prefixes[@]} -eq 0 ]; then
+  echo "No .up.sql migration files found in $migrations_path."
+  exit 1
+fi
+
+# Iterate over prefixes to ensure that there are no gaps and that each prefix
+# has a corresponding .down.sql file. Because the prefixes are sorted, and the
+# index starts at 0, we expect each prefix to be the index plus one.
+for i in "${!prefixes[@]}"; do
+  expected_prefix=$(printf "%06d" $((i+1)))
+
+  if [ "${prefixes[$i]}" != "$expected_prefix" ]; then
+    echo "Error: Missing migration with prefix $expected_prefix."
+    exit 1
+  fi
+
+  # Find the corresponding .down.sql file.
+  base_filename=$(ls "$migrations_path/${prefixes[$i]}"_*.up.sql | \
+      sed -E 's/\.up\.sql//')
+
+  if [ ! -f "$base_filename.down.sql" ]; then
+    echo "Error: Missing .down.sql file for migration $expected_prefix."
+    exit 1
+  fi
+done
+
+echo "All migration files are present and properly numbered."
diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh
@@ -8,6 +8,22 @@ DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
 GOCACHE=`go env GOCACHE`
 GOMODCACHE=`go env GOMODCACHE`
 
+# SQLite doesn't support "BIGINT PRIMARY KEY" for auto-incrementing primary
+# keys, only "INTEGER PRIMARY KEY". Internally it uses 64-bit integers for
+# numbers anyway, independent of the column type. So we can just use
+# "INTEGER PRIMARY KEY" and it will work the same under the hood, giving us
+# auto incrementing 64-bit integers.
+# _BUT_, sqlc will generate Go code with int32 if we use "INTEGER PRIMARY KEY",
+# even though we want int64. So before we run sqlc, we need to patch the
+# source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER
+# PRIMARY KEY".
+echo "Applying SQLite bigint patch..."
+for file in tapdb/sqlc/migrations/*.up.sql; do
+  echo "Patching $file"
+  sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file"
+done
+
+
 echo "Generating sql models and queries in go..."
 
 docker run \
@@ -17,3 +33,9 @@ docker run \
   -v "$DIR/../:/build" \
   -w /build \
   sqlc/sqlc:1.25.0 generate
+
+# Restore the original schema files.
+echo "Restoring SQLite bigint patch..."
+for file in tapdb/sqlc/migrations/*.up.sql.bak; do
+  mv "$file" "${file%.bak}"
+done
diff --git a/tapdb/migrations.go b/tapdb/migrations.go
@@ -22,7 +22,7 @@ const (
 	// daemon.
 	//
 	// NOTE: This MUST be updated when a new migration is added.
-	LatestMigrationVersion = 22
+	LatestMigrationVersion = 23
 )
 
 // MigrationTarget is a functional option that can be passed to applyMigrations
diff --git a/tapdb/postgres.go b/tapdb/postgres.go
@@ -36,8 +36,7 @@ var (
 	// to work with sqlite primarily, and postgres has some differences.
 	postgresSchemaReplacements = map[string]string{
 		"BLOB":                "BYTEA",
-		"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
-		"BIGINT PRIMARY KEY":  "BIGSERIAL PRIMARY KEY",
+		"INTEGER PRIMARY KEY": "BIGSERIAL PRIMARY KEY",
 		"TIMESTAMP":           "TIMESTAMP WITHOUT TIME ZONE",
 		"UNHEX":               "DECODE",
 	}
diff --git a/tapdb/sqlc/migrations/000002_assets.up.sql b/tapdb/sqlc/migrations/000002_assets.up.sql
@@ -3,7 +3,7 @@
 -- information, along with indexing information is stored.
 -- TODO(roasbeef): also store SPV proof?
 CREATE TABLE IF NOT EXISTS chain_txns (
-    txn_id BIGINT PRIMARY KEY,
+    txn_id INTEGER PRIMARY KEY,
 
     txid BLOB UNIQUE NOT NULL,
 
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS chain_txns (
 -- outpoint itself, and also a references to the transaction that _spends_ that
 -- outpoint.
 CREATE TABLE IF NOT EXISTS genesis_points (
-    genesis_id BIGINT PRIMARY KEY,
+    genesis_id INTEGER PRIMARY KEY,
 
     -- TODO(roasbeef): just need the input index here instead?
     prev_out BLOB UNIQUE NOT NULL,
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS genesis_points (
 -- assets that we either created, or bootstrapped from the relevant Base
 -- Universe.
 CREATE TABLE IF NOT EXISTS assets_meta (
-    meta_id BIGINT PRIMARY KEY,
+    meta_id INTEGER PRIMARY KEY,
 
     meta_data_hash BLOB UNIQUE CHECK(length(meta_data_hash) = 32),
 
@@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS assets_meta (
 -- reference the genesis point which is also a necessary component for
 -- computing an asset ID.
 CREATE TABLE IF NOT EXISTS genesis_assets (
-    gen_asset_id BIGINT PRIMARY KEY,
+    gen_asset_id INTEGER PRIMARY KEY,
 
     asset_id BLOB UNIQUE,
 
@@ -72,7 +72,7 @@ CREATE INDEX IF NOT EXISTS asset_ids on genesis_assets(asset_id);
 -- full KeyLocator is stored so we can use these keys without actually storing
 -- the private keys on disk.
 CREATE TABLE IF NOT EXISTS internal_keys (
-    key_id BIGINT PRIMARY KEY,
+    key_id INTEGER PRIMARY KEY,
 
     -- We'll always store the full 33-byte key on disk, to make sure we're
     -- retaining full information.
@@ -88,7 +88,7 @@ CREATE TABLE IF NOT EXISTS internal_keys (
 -- tweaking the base group key by the associated genesis point. This table
 -- references the set of internal keys, and also the genesis_points table.
 CREATE TABLE IF NOT EXISTS asset_groups (
-    group_id BIGINT PRIMARY KEY,
+    group_id INTEGER PRIMARY KEY,
 
     tweaked_group_key BLOB UNIQUE NOT NULL CHECK(length(tweaked_group_key) = 33), 
 
@@ -107,7 +107,7 @@ CREATE TABLE IF NOT EXISTS asset_groups (
 -- asset ID must also be included. This table reference the asset ID it's used
 -- to create as well as the group key that signed the asset in the first place.
 CREATE TABLE IF NOT EXISTS asset_group_witnesses (
-    witness_id BIGINT PRIMARY KEY,
+    witness_id INTEGER PRIMARY KEY,
 
     -- The witness stack can contain either a single Schnorr signature for key
     -- spends of the tweaked group key, or a more complex script witness.
@@ -124,7 +124,7 @@ CREATE TABLE IF NOT EXISTS asset_group_witnesses (
 -- wallet, so the wallet is able to keep track of the amount of sats that are
 -- used to anchor Taproot assets.
 CREATE TABLE IF NOT EXISTS managed_utxos (
-    utxo_id BIGINT PRIMARY KEY,
+    utxo_id INTEGER PRIMARY KEY,
 
     outpoint BLOB UNIQUE NOT NULL,
 
@@ -163,7 +163,7 @@ CREATE TABLE IF NOT EXISTS managed_utxos (
 );
 
 CREATE TABLE IF NOT EXISTS script_keys (
-    script_key_id BIGINT PRIMARY KEY,
+    script_key_id INTEGER PRIMARY KEY,
 
     -- The actual internal key here that we hold the private key for. Applying
     -- the tweak to this gives us the tweaked_script_key.
@@ -184,7 +184,7 @@ CREATE TABLE IF NOT EXISTS script_keys (
 -- asset, along with the sibling taproot hash needed to properly reveal and
 -- spend the asset.
 CREATE TABLE IF NOT EXISTS assets (
-    asset_id BIGINT PRIMARY KEY,
+    asset_id INTEGER PRIMARY KEY,
     
     genesis_id BIGINT NOT NULL REFERENCES genesis_assets(gen_asset_id),
 
@@ -225,7 +225,7 @@ CREATE TABLE IF NOT EXISTS assets (
 -- asset. This then references the script key of an asset, creation a one to
 -- many relationship.
 CREATE TABLE IF NOT EXISTS asset_witnesses (
-    witness_id BIGINT PRIMARY KEY,
+    witness_id INTEGER PRIMARY KEY,
 
     asset_id BIGINT NOT NULL REFERENCES assets(asset_id) ON DELETE CASCADE,
 
@@ -243,7 +243,7 @@ CREATE TABLE IF NOT EXISTS asset_witnesses (
 );
 
 CREATE TABLE IF NOT EXISTS asset_proofs (
-    proof_id BIGINT PRIMARY KEY,
+    proof_id INTEGER PRIMARY KEY,
 
     -- We enforce that this value is unique so we can use an UPSERT to update a
     -- proof file that already exists.
@@ -260,7 +260,7 @@ CREATE TABLE IF NOT EXISTS asset_proofs (
 -- minting transaction which once signed and broadcast will actually create the
 -- assets.
 CREATE TABLE IF NOT EXISTS asset_minting_batches (
-    batch_id BIGINT PRIMARY KEY REFERENCES internal_keys(key_id),
+    batch_id INTEGER PRIMARY KEY REFERENCES internal_keys(key_id),
 
     -- TODO(roasbeef): make into proper enum table or use check to ensure
     -- proper values
@@ -281,7 +281,7 @@ CREATE INDEX IF NOT EXISTS batch_state_lookup on asset_minting_batches (batch_st
 -- asset_seedlings are budding assets: the contain the base asset information
 -- need to create an asset, but doesn't yet have a genesis point.
 CREATE TABLE IF NOT EXISTS asset_seedlings (
-    seedling_id BIGINT PRIMARY KEY,
+    seedling_id INTEGER PRIMARY KEY,
 
     -- TODO(roasbeef): data redundant w/ genesis_assets?
     -- move into asset details table?
diff --git a/tapdb/sqlc/migrations/000003_addrs.up.sql b/tapdb/sqlc/migrations/000003_addrs.up.sql
@@ -2,7 +2,7 @@
 -- a creation time and all the information needed to reconstruct the taproot
 -- output on chain we'll use to send/recv to/from this address.
 CREATE TABLE IF NOT EXISTS addrs (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- version is the version of the Taproot Asset address format.
     version SMALLINT NOT NULL,
diff --git a/tapdb/sqlc/migrations/000005_transfers.up.sql b/tapdb/sqlc/migrations/000005_transfers.up.sql
@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS asset_transfers (
-    id BIGINT PRIMARY KEY, 
+    id INTEGER PRIMARY KEY, 
 
     height_hint INTEGER NOT NULL,
     
@@ -13,7 +13,7 @@ CREATE INDEX IF NOT EXISTS transfer_txn_idx
     ON asset_transfers (anchor_txn_id);
 
 CREATE TABLE IF NOT EXISTS asset_transfer_inputs (
-    input_id BIGINT PRIMARY KEY,
+    input_id INTEGER PRIMARY KEY,
     
     transfer_id BIGINT NOT NULL REFERENCES asset_transfers(id),
     
@@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transfer_inputs_idx
     ON asset_transfer_inputs (transfer_id);
 
 CREATE TABLE IF NOT EXISTS asset_transfer_outputs (
-    output_id BIGINT PRIMARY KEY,
+    output_id INTEGER PRIMARY KEY,
     
     transfer_id BIGINT NOT NULL REFERENCES asset_transfers(id),
     
@@ -74,7 +74,7 @@ CREATE INDEX IF NOT EXISTS proof_locator_hash_index
 -- passive_assets is a table that stores the information needed to
 -- re-anchor a passive asset.
 CREATE TABLE IF NOT EXISTS passive_assets (
-    passive_id BIGINT PRIMARY KEY,
+    passive_id INTEGER PRIMARY KEY,
 
     transfer_id BIGINT NOT NULL REFERENCES asset_transfers(id),
 
diff --git a/tapdb/sqlc/migrations/000006_addr_event.up.sql b/tapdb/sqlc/migrations/000006_addr_event.up.sql
@@ -1,7 +1,7 @@
 -- addr_events stores all events related to inbound (received) assets for
 -- addresses.
 CREATE TABLE IF NOT EXISTS addr_events (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- creation_time is the creation time of this event.
     creation_time TIMESTAMP NOT NULL,
diff --git a/tapdb/sqlc/migrations/000007_universe.up.sql b/tapdb/sqlc/migrations/000007_universe.up.sql
@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS universe_roots (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- For the namespace root, we set the foreign key constraint evaluation to
     -- be deferred until after the database transaction ends. Otherwise, if the
@@ -23,7 +23,7 @@ CREATE INDEX IF NOT EXISTS universe_roots_asset_id_idx ON universe_roots(asset_i
 CREATE INDEX IF NOT EXISTS universe_roots_group_key_idx ON universe_roots(group_key);
 
 CREATE TABLE IF NOT EXISTS universe_leaves (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     asset_genesis_id BIGINT NOT NULL REFERENCES genesis_assets(gen_asset_id),
 
@@ -44,7 +44,7 @@ CREATE INDEX IF NOT EXISTS universe_leaves_key_idx ON universe_leaves(leaf_node_
 CREATE INDEX IF NOT EXISTS universe_leaves_namespace ON universe_leaves(leaf_node_namespace);
 
 CREATE TABLE IF NOT EXISTS universe_servers (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     server_host TEXT UNIQUE NOT NULL,
 
@@ -59,7 +59,7 @@ CREATE TABLE IF NOT EXISTS universe_servers (
 CREATE INDEX IF NOT EXISTS universe_servers_host ON universe_servers(server_host);
 
 CREATE TABLE IF NOT EXISTS universe_events (
-    event_id BIGINT PRIMARY KEY,
+    event_id INTEGER PRIMARY KEY,
 
     event_type VARCHAR NOT NULL CHECK (event_type IN ('SYNC', 'NEW_PROOF', 'NEW_ROOT')),
 
diff --git a/tapdb/sqlc/migrations/000013_universe_fed_proof_sync_log.up.sql b/tapdb/sqlc/migrations/000013_universe_fed_proof_sync_log.up.sql
@@ -1,7 +1,7 @@
 -- This table stores the log of federation universe proof sync attempts. Rows
 -- in this table are specific to a given proof leaf, server, and sync direction.
 CREATE TABLE IF NOT EXISTS federation_proof_sync_log (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- The status of the proof sync attempt.
     status TEXT NOT NULL CHECK(status IN ('pending', 'complete')),
diff --git a/tapdb/sqlc/migrations/000014_multiverse_tree.up.sql b/tapdb/sqlc/migrations/000014_multiverse_tree.up.sql
@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS multiverse_roots (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- For the namespace root, we set the foreign key constraint evaluation to
     -- be deferred until after the database transaction ends. Otherwise, if the
@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS multiverse_roots (
 );
 
 CREATE TABLE IF NOT EXISTS multiverse_leaves (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     multiverse_root_id BIGINT NOT NULL REFERENCES multiverse_roots(id),
 
diff --git a/tapdb/sqlc/migrations/000016_tapscript_trees.up.sql b/tapdb/sqlc/migrations/000016_tapscript_trees.up.sql
@@ -6,7 +6,7 @@ ALTER TABLE asset_minting_batches ADD COLUMN tapscript_sibling BLOB;
 -- This table stores root hashes for tapscript trees, and a flag to ensure that
 -- the stored tree nodes are decoded correctly.
 CREATE TABLE IF NOT EXISTS tapscript_roots (
-        root_id BIGINT PRIMARY KEY,
+        root_id INTEGER PRIMARY KEY,
 
         -- The root hash of a tapscript tree.
         root_hash BLOB NOT NULL UNIQUE CHECK(length(root_hash) = 32),
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS tapscript_roots (
 -- This table stores tapscript nodes, which are tapHashes or tapLeafs. A node
 -- may be included in multiple tapscript trees.
 CREATE TABLE IF NOT EXISTS tapscript_nodes (
-        node_id BIGINT PRIMARY KEY,
+        node_id INTEGER PRIMARY KEY,
 
         -- The serialized tapscript node, which may be a tapHash or tapLeaf.
         raw_node BLOB NOT NULL UNIQUE
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS tapscript_nodes (
 -- This table stores tapscript edges, which link a serialized tapscript node
 -- to a tapscript tree root hash and preserve the node ordering in the tree.
 CREATE TABLE IF NOT EXISTS tapscript_edges (
-        edge_id BIGINT PRIMARY KEY,
+        edge_id INTEGER PRIMARY KEY,
 
         -- The root hash of a tree that includes the referenced tapscript node.
         root_hash_id BIGINT NOT NULL REFERENCES tapscript_roots(root_id),
diff --git a/tapdb/sqlc/migrations/000023_multiverse_tree_re_apply.down.sql b/tapdb/sqlc/migrations/000023_multiverse_tree_re_apply.down.sql
@@ -0,0 +1,3 @@
+DROP INDEX IF EXISTS multiverse_leaves_unique;
+DROP TABLE IF EXISTS multiverse_leaves;
+DROP TABLE IF EXISTS multiverse_roots;
diff --git a/tapdb/sqlc/migrations/000023_multiverse_tree_re_apply.up.sql b/tapdb/sqlc/migrations/000023_multiverse_tree_re_apply.up.sql
diff --git a/tapdb/sqlite.go b/tapdb/sqlite.go

Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ const (`
`22`	`22`	`// daemon.`
`23`	`23`	`//`
`24`	`24`	`// NOTE: This MUST be updated when a new migration is added.`
`25`		`- LatestMigrationVersion = 22`
	`25`	`+ LatestMigrationVersion = 23`
`26`	`26`	`)`
`27`	`27`
`28`	`28`	`// MigrationTarget is a functional option that can be passed to applyMigrations`