Merge pull request #60 from ziggie1984/update-migration-doc

migration: check db versions before migrating any db

Author: guggero

cmd_migrate_db.go

@@ -177,6 +177,14 @@ var (
 		lncfg.NSWalletDB,
 		lncfg.NSNeutrinoDB,
 	}
+
+	// databasesWithMigrations are the databases that have new versions
+	// introduced during their lifetime and therefore need to be up to
+	// date before we can start the migration to SQL backends.
+	databasesWithMigrations = []string{
+		lncfg.NSChannelDB,
+		lncfg.NSTowerClientDB,
+	}
 )
 
 func (x *migrateDBCommand) Execute(_ []string) error {
@@ -205,6 +213,60 @@ func (x *migrateDBCommand) Execute(_ []string) error {
 	// interceptor.
 	ctx := getContext()
 
+	// Before starting any migration, make sure that all existing dbs have
+	// the latest migrations applied so that we do only migrate when all
+	// dbs are up to date. For example it might happen that a user run a
+	// watchtower client earlier and has it switched off since new
+	// migrations have been introduced. In that case we do not start the
+	// migration but first let the user either delete the outdated db or
+	// migrate it by activating the watchtower client again.
+	for _, prefix := range databasesWithMigrations {
+		logger.Infof("Checking db version of database %s", prefix)
+
+		// Check if the database file exists before trying to open it.
+		dbPath := getBoltDBPath(x.Source, prefix, x.Network)
+		if dbPath == "" {
+			return fmt.Errorf("unknown prefix: %s", prefix)
+		}
+
+		// Open and check the database version.
+		srcDb, err := openSourceDb(x.Source, prefix, x.Network, true)
+		if err == kvdb.ErrDbDoesNotExist {
+			// Only skip if it's an optional because it's not
+			// required to run a wtclient or wtserver for example.
+			if optionalDBs[prefix] {
+				logger.Warnf("Skipping checking db version "+
+					"of optional DB %s: not found", prefix)
+
+				continue
+			}
+		}
+		if err != nil {
+			return fmt.Errorf("failed to open source db with "+
+				"prefix `%s` for version check: %w", prefix,
+				err)
+		}
+
+		switch prefix {
+		case lncfg.NSChannelDB:
+			err := checkChannelDBMigrationsApplied(srcDb)
+			if err != nil {
+				srcDb.Close()
+				return err
+			}
+
+		case lncfg.NSTowerClientDB:
+			err := checkWTClientDBMigrationsApplied(srcDb)
+			if err != nil {
+				srcDb.Close()
+				return err
+			}
+		}
+
+		srcDb.Close()
+		logger.Infof("Version check passed for database %s", prefix)
+	}
+
 	for _, prefix := range allDBPrefixes {
 		logger.Infof("Attempting to migrate DB with prefix `%s`", prefix)
 
@@ -344,31 +406,6 @@ func (x *migrateDBCommand) Execute(_ []string) error {
 				prefix, sourceMarker, destMarker)
 		}
 
-		// Check that the source DB has had all its schema migrations
-		// applied before we migrate any of its data. Currently only
-		// migration of the channel.db and the watchtower.db exist.
-		// Check channel.db migrations.
-		if prefix == lncfg.NSChannelDB {
-			logger.Info("Checking DB version of source DB " +
-				"(channel.db)")
-
-			err := checkChannelDBMigrationsApplied(srcDb)
-			if err != nil {
-				return err
-			}
-		}
-
-		// Check watchtower client DB migrations.
-		if prefix == lncfg.NSTowerClientDB {
-			logger.Info("Checking DB version of source DB " +
-				"(wtclient.db)")
-
-			err := checkWTClientDBMigrationsApplied(srcDb)
-			if err != nil {
-				return err
-			}
-		}
-
 		// In case we want to start a new migration we delete the
 		// migration meta db if it exits. This can only be done if
 		// the db is not already successfully migrated otherwise
@@ -826,7 +863,9 @@ func checkWTClientDBMigrationsApplied(db kvdb.Backend) error {
 		return fmt.Errorf("refusing to migrate source database with "+
 			"version %d while latest known DB version is %d; "+
 			"please upgrade the DB before using the data "+
-			"migration tool", version, wtdb.LatestDBMigrationVersion())
+			"migration tool or delete the wtclient.db manually "+
+			"in case you don't plan to use the watchtower client "+
+			"anymore", version, wtdb.LatestDBMigrationVersion())
 	}
 
 	return nil

docs/data-migration.md

@@ -69,7 +69,12 @@ e.g. 200MB
    This version introduced an optimized revocation log storage system that
    reduces the storage footprint. All channels will be automatically migrated
    to this new format when the setting is enabled.
-6. Start `lnd` normally, using the flags mentioned above but not yet changing
+6. If you have ever run the watchtower client you need to make sure before
+   starting the migration that you either activate it again and restart LND so
+   that the db is up to date (the newest migrations have been applied) or you
+   delete the `wtclient.db` manually so that the migration can proceed
+   successfully because you do not plan to use the wtclient anymore.
+7. Start `lnd` normally, using the flags mentioned above but not yet changing
    any database backend related configuration options. Check the log that the
    database schema was migrated successfully, for example: `Checking for
    schema update: latest_version=XX, db_version=XX`. This relates to the 
@@ -79,7 +84,7 @@ e.g. 200MB
    that all LND nodes which want to migrate to the SQL world have the latest db
    modifications in place before migrating to a different DB type. If that is
    not the case the migration will be refused. 
-7. Stop `lnd` again and make sure it isn't started again by accident during the
+8. Stop `lnd` again and make sure it isn't started again by accident during the
    data migration (e.g. disable any `systemd` or other scripts that start/stop
    `lnd`).