scripts+tapdb: do sqlc required replacement in script

Our SQL scripts on disk were neither SQLite nor Postgres compatible, we
used a version suited for sqlc to generate the correct data types we
wanted, then did some type replacements in-memory when actually applying
the migration files.
This is somewhat dangerous as it will lead to situations where if a user
applies a migration file manually, they'll end up with table primary
keys that ARE NOT auto incrementing.

To fix this issue, we make sure that we always write fully SQLite
compatible SQL code and only apply fixes for the sqlc code generation
directly before generating the code.

Author: guggero

scripts/gen_sqlc_docker.sh

@@ -8,6 +8,22 @@ DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
 GOCACHE=`go env GOCACHE`
 GOMODCACHE=`go env GOMODCACHE`
 
+# SQLite doesn't support "BIGINT PRIMARY KEY" for auto-incrementing primary
+# keys, only "INTEGER PRIMARY KEY". Internally it uses 64-bit integers for
+# numbers anyway, independent of the column type. So we can just use
+# "INTEGER PRIMARY KEY" and it will work the same under the hood, giving us
+# auto incrementing 64-bit integers.
+# _BUT_, sqlc will generate Go code with int32 if we use "INTEGER PRIMARY KEY",
+# even though we want int64. So before we run sqlc, we need to patch the
+# source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER
+# PRIMARY KEY".
+echo "Applying SQLite bigint patch..."
+for file in tapdb/sqlc/migrations/*.up.sql; do
+  echo "Patching $file"
+  sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file"
+done
+
+
 echo "Generating sql models and queries in go..."
 
 docker run \
@@ -17,3 +33,9 @@ docker run \
   -v "$DIR/../:/build" \
   -w /build \
   sqlc/sqlc:1.25.0 generate
+
+# Restore the original schema files.
+echo "Restoring SQLite bigint patch..."
+for file in tapdb/sqlc/migrations/*.up.sql.bak; do
+  mv "$file" "${file%.bak}"
+done

tapdb/postgres.go

@@ -36,8 +36,7 @@ var (
 	// to work with sqlite primarily, and postgres has some differences.
 	postgresSchemaReplacements = map[string]string{
 		"BLOB":                "BYTEA",
-		"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
-		"BIGINT PRIMARY KEY":  "BIGSERIAL PRIMARY KEY",
+		"INTEGER PRIMARY KEY": "BIGSERIAL PRIMARY KEY",
 		"TIMESTAMP":           "TIMESTAMP WITHOUT TIME ZONE",
 		"UNHEX":               "DECODE",
 	}

tapdb/sqlc/migrations/000002_assets.up.sql

@@ -3,7 +3,7 @@
 -- information, along with indexing information is stored.
 -- TODO(roasbeef): also store SPV proof?
 CREATE TABLE IF NOT EXISTS chain_txns (
-    txn_id BIGINT PRIMARY KEY,
+    txn_id INTEGER PRIMARY KEY,
 
     txid BLOB UNIQUE NOT NULL,
 
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS chain_txns (
 -- outpoint itself, and also a references to the transaction that _spends_ that
 -- outpoint.
 CREATE TABLE IF NOT EXISTS genesis_points (
-    genesis_id BIGINT PRIMARY KEY,
+    genesis_id INTEGER PRIMARY KEY,
 
     -- TODO(roasbeef): just need the input index here instead?
     prev_out BLOB UNIQUE NOT NULL,
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS genesis_points (
 -- assets that we either created, or bootstrapped from the relevant Base
 -- Universe.
 CREATE TABLE IF NOT EXISTS assets_meta (
-    meta_id BIGINT PRIMARY KEY,
+    meta_id INTEGER PRIMARY KEY,
 
     meta_data_hash BLOB UNIQUE CHECK(length(meta_data_hash) = 32),
 
@@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS assets_meta (
 -- reference the genesis point which is also a necessary component for
 -- computing an asset ID.
 CREATE TABLE IF NOT EXISTS genesis_assets (
-    gen_asset_id BIGINT PRIMARY KEY,
+    gen_asset_id INTEGER PRIMARY KEY,
 
     asset_id BLOB UNIQUE,
 
@@ -72,7 +72,7 @@ CREATE INDEX IF NOT EXISTS asset_ids on genesis_assets(asset_id);
 -- full KeyLocator is stored so we can use these keys without actually storing
 -- the private keys on disk.
 CREATE TABLE IF NOT EXISTS internal_keys (
-    key_id BIGINT PRIMARY KEY,
+    key_id INTEGER PRIMARY KEY,
 
     -- We'll always store the full 33-byte key on disk, to make sure we're
     -- retaining full information.
@@ -88,7 +88,7 @@ CREATE TABLE IF NOT EXISTS internal_keys (
 -- tweaking the base group key by the associated genesis point. This table
 -- references the set of internal keys, and also the genesis_points table.
 CREATE TABLE IF NOT EXISTS asset_groups (
-    group_id BIGINT PRIMARY KEY,
+    group_id INTEGER PRIMARY KEY,
 
     tweaked_group_key BLOB UNIQUE NOT NULL CHECK(length(tweaked_group_key) = 33), 
 
@@ -107,7 +107,7 @@ CREATE TABLE IF NOT EXISTS asset_groups (
 -- asset ID must also be included. This table reference the asset ID it's used
 -- to create as well as the group key that signed the asset in the first place.
 CREATE TABLE IF NOT EXISTS asset_group_witnesses (
-    witness_id BIGINT PRIMARY KEY,
+    witness_id INTEGER PRIMARY KEY,
 
     -- The witness stack can contain either a single Schnorr signature for key
     -- spends of the tweaked group key, or a more complex script witness.
@@ -124,7 +124,7 @@ CREATE TABLE IF NOT EXISTS asset_group_witnesses (
 -- wallet, so the wallet is able to keep track of the amount of sats that are
 -- used to anchor Taproot assets.
 CREATE TABLE IF NOT EXISTS managed_utxos (
-    utxo_id BIGINT PRIMARY KEY,
+    utxo_id INTEGER PRIMARY KEY,
 
     outpoint BLOB UNIQUE NOT NULL,
 
@@ -163,7 +163,7 @@ CREATE TABLE IF NOT EXISTS managed_utxos (
 );
 
 CREATE TABLE IF NOT EXISTS script_keys (
-    script_key_id BIGINT PRIMARY KEY,
+    script_key_id INTEGER PRIMARY KEY,
 
     -- The actual internal key here that we hold the private key for. Applying
     -- the tweak to this gives us the tweaked_script_key.
@@ -184,7 +184,7 @@ CREATE TABLE IF NOT EXISTS script_keys (
 -- asset, along with the sibling taproot hash needed to properly reveal and
 -- spend the asset.
 CREATE TABLE IF NOT EXISTS assets (
-    asset_id BIGINT PRIMARY KEY,
+    asset_id INTEGER PRIMARY KEY,
 
     genesis_id BIGINT NOT NULL REFERENCES genesis_assets(gen_asset_id),
 
@@ -225,7 +225,7 @@ CREATE TABLE IF NOT EXISTS assets (
 -- asset. This then references the script key of an asset, creation a one to
 -- many relationship.
 CREATE TABLE IF NOT EXISTS asset_witnesses (
-    witness_id BIGINT PRIMARY KEY,
+    witness_id INTEGER PRIMARY KEY,
 
     asset_id BIGINT NOT NULL REFERENCES assets(asset_id) ON DELETE CASCADE,
 
@@ -243,7 +243,7 @@ CREATE TABLE IF NOT EXISTS asset_witnesses (
 );
 
 CREATE TABLE IF NOT EXISTS asset_proofs (
-    proof_id BIGINT PRIMARY KEY,
+    proof_id INTEGER PRIMARY KEY,
 
     -- We enforce that this value is unique so we can use an UPSERT to update a
     -- proof file that already exists.
@@ -260,7 +260,7 @@ CREATE TABLE IF NOT EXISTS asset_proofs (
 -- minting transaction which once signed and broadcast will actually create the
 -- assets.
 CREATE TABLE IF NOT EXISTS asset_minting_batches (
-    batch_id BIGINT PRIMARY KEY REFERENCES internal_keys(key_id),
+    batch_id INTEGER PRIMARY KEY REFERENCES internal_keys(key_id),
 
     -- TODO(roasbeef): make into proper enum table or use check to ensure
     -- proper values
@@ -281,7 +281,7 @@ CREATE INDEX IF NOT EXISTS batch_state_lookup on asset_minting_batches (batch_st
 -- asset_seedlings are budding assets: the contain the base asset information
 -- need to create an asset, but doesn't yet have a genesis point.
 CREATE TABLE IF NOT EXISTS asset_seedlings (
-    seedling_id BIGINT PRIMARY KEY,
+    seedling_id INTEGER PRIMARY KEY,
 
     -- TODO(roasbeef): data redundant w/ genesis_assets?
     -- move into asset details table?

tapdb/sqlc/migrations/000003_addrs.up.sql

@@ -2,7 +2,7 @@
 -- a creation time and all the information needed to reconstruct the taproot
 -- output on chain we'll use to send/recv to/from this address.
 CREATE TABLE IF NOT EXISTS addrs (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- version is the version of the Taproot Asset address format.
     version SMALLINT NOT NULL,

tapdb/sqlc/migrations/000005_transfers.up.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS asset_transfers (
-    id BIGINT PRIMARY KEY, 
+    id INTEGER PRIMARY KEY, 
 
     height_hint INTEGER NOT NULL,
 
@@ -13,7 +13,7 @@ CREATE INDEX IF NOT EXISTS transfer_txn_idx
     ON asset_transfers (anchor_txn_id);
 
 CREATE TABLE IF NOT EXISTS asset_transfer_inputs (
-    input_id BIGINT PRIMARY KEY,
+    input_id INTEGER PRIMARY KEY,
 
     transfer_id BIGINT NOT NULL REFERENCES asset_transfers(id),
 
@@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transfer_inputs_idx
     ON asset_transfer_inputs (transfer_id);
 
 CREATE TABLE IF NOT EXISTS asset_transfer_outputs (
-    output_id BIGINT PRIMARY KEY,
+    output_id INTEGER PRIMARY KEY,
 
     transfer_id BIGINT NOT NULL REFERENCES asset_transfers(id),
 
@@ -74,7 +74,7 @@ CREATE INDEX IF NOT EXISTS proof_locator_hash_index
 -- passive_assets is a table that stores the information needed to
 -- re-anchor a passive asset.
 CREATE TABLE IF NOT EXISTS passive_assets (
-    passive_id BIGINT PRIMARY KEY,
+    passive_id INTEGER PRIMARY KEY,
 
     transfer_id BIGINT NOT NULL REFERENCES asset_transfers(id),
 

tapdb/sqlc/migrations/000006_addr_event.up.sql

@@ -1,7 +1,7 @@
 -- addr_events stores all events related to inbound (received) assets for
 -- addresses.
 CREATE TABLE IF NOT EXISTS addr_events (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- creation_time is the creation time of this event.
     creation_time TIMESTAMP NOT NULL,

tapdb/sqlc/migrations/000007_universe.up.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS universe_roots (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- For the namespace root, we set the foreign key constraint evaluation to
     -- be deferred until after the database transaction ends. Otherwise, if the
@@ -23,7 +23,7 @@ CREATE INDEX IF NOT EXISTS universe_roots_asset_id_idx ON universe_roots(asset_i
 CREATE INDEX IF NOT EXISTS universe_roots_group_key_idx ON universe_roots(group_key);
 
 CREATE TABLE IF NOT EXISTS universe_leaves (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     asset_genesis_id BIGINT NOT NULL REFERENCES genesis_assets(gen_asset_id),
 
@@ -44,7 +44,7 @@ CREATE INDEX IF NOT EXISTS universe_leaves_key_idx ON universe_leaves(leaf_node_
 CREATE INDEX IF NOT EXISTS universe_leaves_namespace ON universe_leaves(leaf_node_namespace);
 
 CREATE TABLE IF NOT EXISTS universe_servers (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     server_host TEXT UNIQUE NOT NULL,
 
@@ -59,7 +59,7 @@ CREATE TABLE IF NOT EXISTS universe_servers (
 CREATE INDEX IF NOT EXISTS universe_servers_host ON universe_servers(server_host);
 
 CREATE TABLE IF NOT EXISTS universe_events (
-    event_id BIGINT PRIMARY KEY,
+    event_id INTEGER PRIMARY KEY,
 
     event_type VARCHAR NOT NULL CHECK (event_type IN ('SYNC', 'NEW_PROOF', 'NEW_ROOT')),
 

tapdb/sqlc/migrations/000013_universe_fed_proof_sync_log.up.sql

@@ -1,7 +1,7 @@
 -- This table stores the log of federation universe proof sync attempts. Rows
 -- in this table are specific to a given proof leaf, server, and sync direction.
 CREATE TABLE IF NOT EXISTS federation_proof_sync_log (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- The status of the proof sync attempt.
     status TEXT NOT NULL CHECK(status IN ('pending', 'complete')),

tapdb/sqlc/migrations/000014_multiverse_tree.up.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS multiverse_roots (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- For the namespace root, we set the foreign key constraint evaluation to
     -- be deferred until after the database transaction ends. Otherwise, if the
@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS multiverse_roots (
 );
 
 CREATE TABLE IF NOT EXISTS multiverse_leaves (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     multiverse_root_id BIGINT NOT NULL REFERENCES multiverse_roots(id),
 

tapdb/sqlc/migrations/000016_tapscript_trees.up.sql

@@ -6,7 +6,7 @@ ALTER TABLE asset_minting_batches ADD COLUMN tapscript_sibling BLOB;
 -- This table stores root hashes for tapscript trees, and a flag to ensure that
 -- the stored tree nodes are decoded correctly.
 CREATE TABLE IF NOT EXISTS tapscript_roots (
-        root_id BIGINT PRIMARY KEY,
+        root_id INTEGER PRIMARY KEY,
 
         -- The root hash of a tapscript tree.
         root_hash BLOB NOT NULL UNIQUE CHECK(length(root_hash) = 32),
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS tapscript_roots (
 -- This table stores tapscript nodes, which are tapHashes or tapLeafs. A node
 -- may be included in multiple tapscript trees.
 CREATE TABLE IF NOT EXISTS tapscript_nodes (
-        node_id BIGINT PRIMARY KEY,
+        node_id INTEGER PRIMARY KEY,
 
         -- The serialized tapscript node, which may be a tapHash or tapLeaf.
         raw_node BLOB NOT NULL UNIQUE
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS tapscript_nodes (
 -- This table stores tapscript edges, which link a serialized tapscript node
 -- to a tapscript tree root hash and preserve the node ordering in the tree.
 CREATE TABLE IF NOT EXISTS tapscript_edges (
-        edge_id BIGINT PRIMARY KEY,
+        edge_id INTEGER PRIMARY KEY,
 
         -- The root hash of a tree that includes the referenced tapscript node.
         root_hash_id BIGINT NOT NULL REFERENCES tapscript_roots(root_id),