sqlite: add missing foreign keys

Author: n8mgr

.changeset/added_missing_block_foreign_key_indices.md

@@ -0,0 +1,5 @@
+---
+default: patch
+---
+
+# Added missing block foreign key indices.

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/library/golang:1.23 AS builder
+FROM docker.io/library/golang:1.24 AS builder
 
 WORKDIR /explored
 

go.mod

@@ -1,8 +1,6 @@
 module go.sia.tech/explored
 
-go 1.23.2
-
-toolchain go1.24.2
+go 1.24.2
 
 require (
 	github.com/google/go-cmp v0.7.0

persist/sqlite/init.go

@@ -16,47 +16,70 @@ import (
 //go:embed init.sql
 var initDatabase string
 
+func initializeSettings(tx *txn, target int64) error {
+	_, err := tx.Exec(`INSERT INTO global_settings (id, db_version) VALUES (0, ?)`, target)
+	return err
+}
+
 func (s *Store) initNewDatabase(tx *txn, target int64) error {
 	if _, err := tx.Exec(initDatabase); err != nil {
 		return fmt.Errorf("failed to initialize database: %w", err)
-	} else if err := setDBVersion(tx, target); err != nil {
-		return fmt.Errorf("failed to set initial database version: %w", err)
+	} else if err := initializeSettings(tx, target); err != nil {
+		return fmt.Errorf("failed to initialize global settings: %w", err)
 	}
 	return nil
 }
 
 func (s *Store) upgradeDatabase(current, target int64) error {
-	log := s.log.Named("migrations")
-	log.Info("migrating database", zap.Int64("current", current), zap.Int64("target", target))
+	log := s.log.Named("migrations").With(zap.Int64("target", target))
+	for ; current < target; current++ {
+		version := current + 1 // initial schema is version 1, migration 0 is version 2, etc.
+		log := log.With(zap.Int64("version", version))
+		start := time.Now()
+		fn := migrations[current-1]
+		err := s.transaction(func(tx *txn) error {
+			if _, err := tx.Exec("PRAGMA defer_foreign_keys=ON"); err != nil {
+				return fmt.Errorf("failed to enable foreign key deferral: %w", err)
+			} else if err := fn(tx, log); err != nil {
+				return err
+			} else if err := foreignKeyCheck(tx, log); err != nil {
+				return fmt.Errorf("failed foreign key check: %w", err)
+			}
+			return setDBVersion(tx, version)
+		})
+		if err != nil {
+			return fmt.Errorf("migration %d failed: %w", version, err)
+		}
+		log.Info("migration complete", zap.Duration("elapsed", time.Since(start)))
+	}
+	return nil
+}
 
-	// disable foreign key constraints during migration
-	if _, err := s.db.Exec("PRAGMA foreign_keys = OFF"); err != nil {
-		return fmt.Errorf("failed to disable foreign key constraints: %w", err)
+func foreignKeyCheck(txn *txn, log *zap.Logger) error {
+	rows, err := txn.Query("PRAGMA foreign_key_check")
+	if err != nil {
+		return fmt.Errorf("failed to run foreign key check: %w", err)
 	}
-	defer func() {
-		// re-enable foreign key constraints
-		if _, err := s.db.Exec("PRAGMA foreign_keys = ON"); err != nil {
-			log.Panic("failed to enable foreign key constraints", zap.Error(err))
-		}
-	}()
+	defer rows.Close()
+	var hasErrors bool
+	for rows.Next() {
+		var table string
+		var rowid sql.NullInt64
+		var fkTable string
+		var fkRowid sql.NullInt64
 
-	return s.transaction(func(tx *txn) error {
-		for _, fn := range migrations[current-1:] {
-			current++
-			start := time.Now()
-			if err := fn(tx, log.With(zap.Int64("version", current))); err != nil {
-				return fmt.Errorf("failed to migrate database to version %v: %w", current, err)
-			}
-			// check that no foreign key constraints were violated
-			if err := tx.QueryRow("PRAGMA foreign_key_check").Scan(); !errors.Is(err, sql.ErrNoRows) {
-				return fmt.Errorf("foreign key constraints are not satisfied")
-			}
-			log.Debug("migration complete", zap.Int64("current", current), zap.Int64("target", target), zap.Duration("elapsed", time.Since(start)))
+		if err := rows.Scan(&table, &rowid, &fkTable, &fkRowid); err != nil {
+			return fmt.Errorf("failed to scan foreign key check result: %w", err)
 		}
-
-		// set the final database version
-		return setDBVersion(tx, target)
-	})
+		hasErrors = true
+		log.Error("foreign key constraint violated", zap.String("table", table), zap.Int64("rowid", rowid.Int64), zap.String("fkTable", fkTable), zap.Int64("fkRowid", fkRowid.Int64))
+	}
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("failed to iterate foreign key check results: %w", err)
+	} else if hasErrors {
+		return errors.New("foreign key constraint violated")
+	}
+	return nil
 }
 
 func (s *Store) init() error {

persist/sqlite/init.sql

@@ -57,7 +57,7 @@ CREATE TABLE siacoin_elements (
 	address BLOB NOT NULL,
 	value BLOB NOT NULL
 );
-
+CREATE INDEX siacoin_elements_blocks_id_index ON siacoin_elements(block_id);
 CREATE INDEX siacoin_elements_maturity_height_index ON siacoin_elements(maturity_height);
 CREATE INDEX siacoin_elements_output_id_index ON siacoin_elements(output_id);
 CREATE INDEX siacoin_elements_address_spent_index ON siacoin_elements(address, spent_index);
@@ -74,7 +74,7 @@ CREATE TABLE siafund_elements (
 	address BLOB NOT NULL,
 	value BLOB NOT NULL
 );
-
+CREATE INDEX siafund_elements_blocks_id_index ON siafund_elements(block_id);
 CREATE INDEX siafund_elements_output_id_index ON siafund_elements(output_id);
 CREATE INDEX siafund_elements_address_spent_index ON siafund_elements(address, spent_index);
 
@@ -95,6 +95,7 @@ CREATE TABLE file_contract_elements (
 	revision_number BLOB NOT NULL,
 	UNIQUE(contract_id, revision_number)
 );
+CREATE INDEX file_contract_elements_blocks_id_index ON file_contract_elements(block_id);
 CREATE INDEX file_contract_elements_contract_id_revision_number_index ON file_contract_elements(contract_id, revision_number);
 
 CREATE TABLE last_contract_revision (
@@ -116,6 +117,7 @@ CREATE TABLE last_contract_revision (
 
 	contract_element_id INTEGER UNIQUE REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL
 );
+CREATE INDEX last_contract_revision_confirmation_block_id_index ON last_contract_revision(confirmation_block_id);
 
 CREATE TABLE file_contract_valid_proof_outputs (
 	contract_id INTEGER REFERENCES file_contract_elements(id) ON DELETE CASCADE NOT NULL,
@@ -145,7 +147,6 @@ CREATE TABLE miner_payouts (
 	output_id INTEGER REFERENCES siacoin_elements(id) ON DELETE CASCADE NOT NULL,
 	UNIQUE(block_id, block_order)
 );
-
 CREATE INDEX miner_payouts_block_id_index ON miner_payouts(block_id);
 
 CREATE TABLE transactions (
@@ -457,6 +458,7 @@ CREATE TABLE v2_file_contract_elements (
 
     UNIQUE(contract_id, revision_number)
 );
+CREATE INDEX v2_file_contract_elements_blocks_id_index ON v2_file_contract_elements(block_id);
 CREATE INDEX v2_file_contract_elements_contract_id_revision_number_index ON v2_file_contract_elements(contract_id, revision_number);
 
 CREATE TABLE v2_last_contract_revision (
@@ -476,6 +478,7 @@ CREATE TABLE v2_last_contract_revision (
 
     contract_element_id INTEGER UNIQUE REFERENCES v2_file_contract_elements(id) ON DELETE CASCADE NOT NULL
 );
+CREATE INDEX v2_last_contract_revision_confirmation_block_id_index ON v2_last_contract_revision(confirmation_block_id);
 
 CREATE TABLE host_info (
     public_key BLOB PRIMARY KEY NOT NULL,
@@ -590,6 +593,3 @@ CREATE TABLE host_info_v2_netaddresses(
 
 CREATE INDEX host_info_v2_netaddresses_public_key ON host_info_v2_netaddresses(public_key);
 CREATE INDEX host_info_v2_netaddresses_address ON host_info_v2_netaddresses(address);
-
--- initialize the global settings table
-INSERT INTO global_settings (id, db_version) VALUES (0, 0); -- should not be changed

persist/sqlite/migrations.go

@@ -2,4 +2,19 @@ package sqlite
 
 import "go.uber.org/zap"
 
-var migrations []func(tx *txn, log *zap.Logger) error
+func migrateV2(txn *txn, _ *zap.Logger) error {
+	const createForeignKeyIndices = `
+CREATE INDEX siacoin_elements_blocks_id_index ON siacoin_elements(block_id);
+CREATE INDEX siafund_elements_blocks_id_index ON siafund_elements(block_id);
+CREATE INDEX file_contract_elements_blocks_id_index ON file_contract_elements(block_id);
+CREATE INDEX last_contract_revision_confirmation_block_id_index ON last_contract_revision(confirmation_block_id);
+CREATE INDEX v2_file_contract_elements_blocks_id_index ON v2_file_contract_elements(block_id);
+CREATE INDEX v2_last_contract_revision_confirmation_block_id_index ON v2_last_contract_revision(confirmation_block_id);
+`
+	_, err := txn.Exec(createForeignKeyIndices)
+	return err
+}
+
+var migrations = []func(tx *txn, log *zap.Logger) error{
+	migrateV2,
+}