v7 to v9 config migration script

node_management/migration/.gitignore

@@ -0,0 +1,13 @@
+# Ignore all files in the ruff cache directory
+**/.ruff_cache
+
+**/build/
+
+# egg-info
+**/*.egg-info/
+
+# python cache 
+**/__pycache__/
+
+# pytest
+**/.hypothesis/

node_management/migration/README.md

@@ -0,0 +1,82 @@
+
+### Usage Example
+
+alias rpc-v8=~/Documents/share/repo/smr-moonshot-testnet/target/devopt/rpc_node
+alias rpc-v9=~/Documents/share/repo/smr-moonshot/target/release/rpc_node
+
+alias supra-v8="~/Documents/share/repo/smr-moonshot-testnet/target/devopt/supra"
+alias supra-v9="~/Documents/share/repo/smr-moonshot/target/release/supra"
+
+
+```sh
+pip install .
+
+# Migrate rpc config from v7 to v9
+migrate-config rpc -p v7-v9  -f config.toml -t config.toml
+# Migrate db from v7 to v8
+rpc-v8 migrate-db config.toml
+# Migrate db from v8 to v9
+rpc-v9 migrate-db config.toml
+
+# Migrate cli profile from v7 to v8
+supra-v8 migrate --network localnet
+cp validator_identity.pem node_identity.pem
+# Migrate cli profile from v8 to v9
+supra-v9 profile migrate
+
+# Migrate smr_settings from v7 to v9
+migrate-config smr -p v7-v9 -f smr_settings.toml -t smr_settings.toml
+# Migrate db from v7 to v9
+supra-v9 data migrate -p smr_settings.toml
+```
+
+#### v9 template
+https://testnet-snapshot.supra.com/configs/config_v9.0.7.toml
+https://testnet-snapshot.supra.com/configs/smr_settings.toml
+
+#### v8 template
+https://testnet-snapshot.supra.com/configs/config_v8.0.2.toml
+
+#### v7 template
+https://mainnet-data.supra.com/configs/config.toml
+
+
+
+### Migrate from v7 to v9
+
+
+#### rpc db migration
+
+(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v8 migrate-db config.toml
+  [======================================================================================================================================================================] 100/100MigrationReport { drop_cf: ["tx_block_info"], migrate_kv: {"tx_block_info__txn_hash_to_block_hash": 8537} }
+
+
+(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/rpc_0$ rpc-v9 migrate-db config.toml
+Counting the number of entries to remove from prune_index...
+Cleaning up prune index: [00:00:00] ████████████████████ 0/0 00:00:00                                                                                                             Counting the number of entries in block_to_transaction...
+Migrating block_to_transaction: [00:00:00] ████████████████████ 8537/8537 00:00:00                                                                                                dropped:
+- block_to_tx
+migrated:
+  block_to_tx -> block_to_tx_ordered: Migrated 8537 records, up to 239 block height
+databases_checked:
+- chain_store
+- archive
+
+
+#### smr db migration
+
+(e2e-tests) kaiqichen@Mac:~/Documents/share/repo/smr-moonshot-mainnet/remote_env/Logs/node_0$ supra-v9 data migrate -p smr_settings.toml
+Counting the number of entries in certified_block...
+Migrating certified_block to certified_block_dehydrated: [00:00:00] ████████████████████ 69/69 00:00:00                                                                                       Counting the number of entries in uncommitted_block...
+Preparing to clean up uncommitted_block: [00:00:00] ████████████████████ 74/74 00:00:00
+Cleaning up uncommitted_block: [00:00:00] ████████████████████ 4/4 00:00:00                                                                                                                   Counting the number of entries in certified_block...
+Counting the number of entries in certified_block_dehydrated...
+Counting the number of entries in uncommitted_block...
+Counting the number of entries in qc...
+Verifying certified_block_dehydrated: [00:00:00] ████████████████████ 70/70 00:00:00                                                                                                          Counting the number of entries to remove from prune_index...
+Cleaning up prune index: [00:00:00] ████████████████████ 70/70 00:00:00                                                                                                                       dropped:
+- certified_block
+migrated:
+  certified_block -> certified_block_dehydrated: Migrated 70 records, up to 244 block height
+databases_checked:
+- chain_store

node_management/migration/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "migrate-config"
+version = "0.1.0"
+description = "Unified CLI tool to migrate Supra RPC and SMR configs."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "tomlkit>=0.13.2",
+    "click>=8.0.0"
+]
+
+[project.scripts]
+migrate-config = "cli.main:main"
+
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"

node_management/migration/src/cli/main.py

@@ -0,0 +1,60 @@
+import click
+from rpc_config.migrate_path import RPC_CONFIG_MIGRATE_PATH
+from rpc_config.migrate_path import run_migration as migrate_rpc_config
+from smr_settings.migrate_path import SMR_SETTINGS_MIGRATE_PATH
+from smr_settings.migrate_path import run_migration as migrate_smr_config
+
+
+@click.group()
+def main():
+    """Migration CLI for Supra configs."""
+
+
+@main.command()
+@click.option(
+    "--migrate-path",
+    "-p",
+    required=True,
+    type=click.Choice(RPC_CONFIG_MIGRATE_PATH, case_sensitive=True),
+    help=f"Migration path (choices: {', '.join(RPC_CONFIG_MIGRATE_PATH)})",
+)
+@click.option(
+    "--from-file",
+    "-f",
+    required=True,
+    type=click.Path(exists=True),
+    help="Source config file",
+)
+@click.option(
+    "--to-file", "-t", required=True, type=click.Path(), help="Output config file"
+)
+def rpc(migrate_path, from_file, to_file):
+    """Migrate RPC config."""
+    migrate_rpc_config(migrate_path, from_file, to_file)
+
+
+@main.command()
+@click.option(
+    "--migrate-path",
+    "-p",
+    required=True,
+    type=click.Choice(SMR_SETTINGS_MIGRATE_PATH, case_sensitive=True),
+    help=f"Migration path (choices: {', '.join(SMR_SETTINGS_MIGRATE_PATH)})",
+)
+@click.option(
+    "--from-file",
+    "-f",
+    required=True,
+    type=click.Path(exists=True),
+    help="Source config file",
+)
+@click.option(
+    "--to-file", "-t", required=True, type=click.Path(), help="Output config file"
+)
+def smr(migrate_path, from_file, to_file):
+    """Migrate SMR config."""
+    migrate_smr_config(migrate_path, from_file, to_file)
+
+
+if __name__ == "__main__":
+    main()

node_management/migration/src/common/migration.py

@@ -0,0 +1,91 @@
+
+import typing as ty
+import tomlkit
+from copy import deepcopy
+from . import utils
+
+class MigrationPathSet:
+    """
+    Base class for migration paths.
+    """
+    def __init__(self, migrate_paths: ty.Dict[str, ty.List[ty.Callable]]):
+        self.migrate_paths = migrate_paths
+
+    def get_versions(self, key: str) -> ty.Tuple[str, str]:
+        """Split the key into from_version and to_version."""
+        if key not in self.migrate_paths:
+            raise ValueError(f"Invalid key: {key}")
+        from_version, to_version = key.split('-', 1)
+        return from_version, to_version
+
+    def get_migration_functions(self, key: str) -> ty.List[ty.Callable]:
+        """Get the migration functions for the given key."""
+        if key not in self.migrate_paths:
+            raise ValueError(f"Unknown migration path: {key}")
+        return self.migrate_paths[key]
+
+
+class Migration:
+    """
+    Top level migration class that handles the migration of config files.
+    """
+    def __init__(self, migrate_path: ty.Dict[str, ty.List[ty.Callable]]):
+        self.migrate_path = MigrationPathSet(migrate_path)
+
+
+    def migrate_config(self, migrate_choice: str, from_path: str, to_path: str):
+
+        migrate_functions = self.migrate_path.get_migration_functions(migrate_choice)
+
+        from_version, to_version = self.migrate_path.get_versions(migrate_choice)
+        default_backup_path = f"{from_path}_{from_version}.bak"
+        if from_path == to_path:
+            print(f"Warning: The source and destination paths are the same ({from_path}).")
+            print(
+                f"A backup of your original config will be saved to: {default_backup_path}"
+            )
+            confirm = input(
+                "This will overwrite your original config file. Continue? [y/N]: "
+            )
+            if confirm.lower() != "y":
+                print("Aborted by user.")
+                return
+
+        with open(from_path, "r") as f:
+            toml_data = tomlkit.parse(f.read())
+
+        # Backup old config
+        original_toml_data = deepcopy(toml_data)
+
+        for fn in migrate_functions:
+            print(f"Running migration function: {fn.__name__}")
+            fn(toml_data)
+
+        print(f"Backing up old config to {default_backup_path}")
+        tomlkit.dump(original_toml_data, open(default_backup_path, "w"))
+
+        # Write new config
+        print(f"Writing new config to {to_path}")
+        with open(to_path, "w") as f:
+            f.write(tomlkit.dumps(toml_data))
+
+        # Print the diff
+        from_str = tomlkit.dumps(original_toml_data).splitlines(keepends=True)
+        to_str = tomlkit.dumps(toml_data).splitlines(keepends=True)
+
+        utils.unified_diff(
+            from_str,
+            to_str,
+            fromfile=from_version,
+            tofile=to_version,
+        )
+
+        print(
+            f"""
+        Config migrated from {from_path} to {to_path}.
+        Please double check above for the diff between old and new config.
+        Please ensure to use the new config file for target binary version.
+        NOTE: the comments may not be preserved in the new config file, so 
+        you would need to fix the comments manually.
+        """
+        )

node_management/migration/src/common/utils.py

@@ -0,0 +1,28 @@
+import difflib
+import typing as ty
+
+def unified_diff(from_str, to_str, fromfile, tofile):
+    diff = difflib.unified_diff(from_str, to_str, fromfile=fromfile, tofile=tofile)
+    __print_colored_diff(diff)
+
+
+def __print_colored_diff(diff):
+    # The color is added here manually using ANSI escape codes.
+    for line in diff:
+        if line.startswith("+") and not line.startswith("+++"):
+            print(f"\033[32m{line}\033[0m", end="")  # Green for additions
+        elif line.startswith("-") and not line.startswith("---"):
+            print(f"\033[31m{line}\033[0m", end="")  # Red for deletions
+        elif line.startswith("@@"):
+            print(f"\033[36m{line}\033[0m", end="")  # Cyan for hunk headers
+        else:
+            print(line, end="")
+
+
+def print_with_checkmark(message):
+    """
+    Print a message with a checkmark.
+    """
+    print(f"✓ {message}")
+
+