Modify harvester pool - update migrate for introducer - Merge pull request #198 from Chia-Network/ms.migrate-intro

Support not migrating some variables

Author: hoffmang9

electron-ui/package-lock.json

@@ -1,6 +1,6 @@
-import argparse
 import importlib
 import pathlib
+from argparse import Namespace, ArgumentParser
 
 from src import __version__
 from src.util.default_root import DEFAULT_ROOT_PATH
@@ -9,8 +9,8 @@
 SUBCOMMANDS = ["init", "generate", "show", "start", "stop", "version", "netspace"]
 
 
-def create_parser():
-    parser = argparse.ArgumentParser(
+def create_parser() -> ArgumentParser:
+    parser: ArgumentParser = ArgumentParser(
         description="Manage chia blockchain infrastructure (%s)." % __version__,
         epilog="Try 'chia start node', 'chat netspace -d 48', or 'chia show -s'.",
     )
@@ -38,7 +38,7 @@ def create_parser():
     return parser
 
 
-def chia(args, parser):
+def chia(args: Namespace, parser: ArgumentParser):
     return args.function(args, parser)
 
 

src/cmds/chia.py

@@ -1,9 +1,12 @@
 import os
 import shutil
 
+from argparse import Namespace, ArgumentParser
+from typing import List, Tuple, Dict, Any
 from blspy import ExtendedPrivateKey
 from src.types.BLSSignature import BLSPublicKey
 from src.consensus.coinbase import create_puzzlehash_for_pk
+from src.util.config import unflatten_properties
 from pathlib import Path
 
 from src.util.config import (
@@ -19,19 +22,26 @@
 from src.ssl.create_ssl import generate_selfsigned_cert
 
 
-def make_parser(parser):
+def make_parser(parser: ArgumentParser):
     parser.set_defaults(function=init)
 
 
-def dict_add_new_default(updated, default):
+def dict_add_new_default(
+    updated: Dict, default: Dict, do_not_migrate_keys: Dict[str, Any]
+):
     for k, v in default.items():
         if isinstance(v, dict) and k in updated:
-            dict_add_new_default(updated[k], default[k])
-        elif k not in updated:
-            updated[k] = default[k]
+            # If there is an intermediate key with empty string value, do not migrate all decendants
+            if do_not_migrate_keys.get(k, None) == "":
+                do_not_migrate_keys[k] = v
+            dict_add_new_default(updated[k], default[k], do_not_migrate_keys.get(k, {}))
+        elif k not in updated or k in do_not_migrate_keys:
+            updated[k] = v
 
 
-def migrate_from(old_root, new_root, manifest):
+def migrate_from(
+    old_root: Path, new_root: Path, manifest: List[str], do_not_migrate_keys: List[str]
+):
     """
     Copy all the files in "manifest" to the new config directory.
     """
@@ -55,26 +65,28 @@ def migrate_from(old_root, new_root, manifest):
             not_found.append(f)
             print(f"{old_path} not found, skipping")
     # update config yaml with new keys
-    config = load_config(new_root, "config.yaml")
-    config_str = initial_config_file("config.yaml")
-    default_config = yaml.safe_load(config_str)
-    dict_add_new_default(config, default_config)
+    config: Dict = load_config(new_root, "config.yaml")
+    config_str: str = initial_config_file("config.yaml")
+    default_config: Dict = yaml.safe_load(config_str)
+    flattened_keys = unflatten_properties({k: "" for k in do_not_migrate_keys})
+    dict_add_new_default(config, default_config, flattened_keys)
 
     save_config(new_root, "config.yaml", config)
+
     # migrate plots
     # for now, we simply leave them where they are
     # and make what may have been relative paths absolute
     if "config/trusted.key" in not_found or "config/trusted.key" in not_found:
         initialize_ssl(new_root)
 
-    plots_config = load_config(new_root, "plots.yaml")
+    plots_config: Dict = load_config(new_root, "plots.yaml")
 
     plot_root = (
         load_config(new_root, "config.yaml").get("harvester", {}).get("plot_root", ".")
     )
 
-    old_plots_root = path_from_root(old_root, plot_root)
-    new_plots_root = path_from_root(new_root, plot_root)
+    old_plots_root: Path = path_from_root(old_root, plot_root)
+    new_plots_root: Path = path_from_root(new_root, plot_root)
 
     old_plot_paths = plots_config.get("plots", {})
     if len(old_plot_paths) == 0:
@@ -83,13 +95,13 @@ def migrate_from(old_root, new_root, manifest):
 
     print("\nmigrating plots.yaml")
 
-    new_plot_paths = {}
+    new_plot_paths: Dict = {}
     for path, values in old_plot_paths.items():
         old_path_full = path_from_root(old_plots_root, path)
         new_path_relative = make_path_relative(old_path_full, new_plots_root)
         print(f"rewriting {path}\n as {new_path_relative}")
         new_plot_paths[str(new_path_relative)] = values
-    plots_config_new = {"plots": new_plot_paths}
+    plots_config_new: Dict = {"plots": new_plot_paths}
     save_config(new_root, "plots.yaml", plots_config_new)
     print("\nUpdated plots.yaml to point to where your existing plots are.")
     print(
@@ -117,7 +129,7 @@ def migrate_from(old_root, new_root, manifest):
     return 1
 
 
-def initialize_ssl(root_path):
+def initialize_ssl(root_path: Path):
     cert, key = generate_selfsigned_cert()
     path_crt = config_path_for_filename(root_path, "trusted.crt")
     path_key = config_path_for_filename(root_path, "trusted.key")
@@ -127,32 +139,39 @@ def initialize_ssl(root_path):
         f.write(key)
 
 
-def init(args, parser):
+def init(args: Namespace, parser: ArgumentParser):
     return chia_init(args)
 
 
-def chia_init(args):
-    root_path = args.root_path
+def chia_init(args: Namespace):
+    root_path: Path = args.root_path
     print(f"migrating to {root_path}")
     if root_path.is_dir():
         print(f"{root_path} already exists, no action taken")
         return -1
 
-    MANIFEST = [
+    # These are the config keys that will not be migrated, and instead the default is used
+    DO_NOT_MIGRATE_KEYS: List[str] = [
+        "full_node.introducer_peer",
+        "wallet.introducer_peer",
+    ]
+
+    # These are the files that will be migrated
+    MANIFEST: List[str] = [
         "config/config.yaml",
         "config/plots.yaml",
         "config/keys.yaml",
         "config/trusted.crt",
         "config/trusted.key",
     ]
 
-    PATH_MANIFEST_LIST = [
+    PATH_MANIFEST_LIST: List[Tuple[Path, List[str]]] = [
         (Path(os.path.expanduser("~/.chia/beta-%s" % _)), MANIFEST)
         for _ in ["1.0b3", "1.0b2", "1.0b1"]
     ]
 
     for old_path, manifest in PATH_MANIFEST_LIST:
-        r = migrate_from(old_path, root_path, manifest)
+        r = migrate_from(old_path, root_path, manifest, DO_NOT_MIGRATE_KEYS)
         if r:
             break
     else:

src/cmds/init.py

@@ -161,7 +161,7 @@ async def new_challenge(self, new_challenge: harvester_protocol.NewChallenge):
             )
 
         loop = asyncio.get_running_loop()
-        executor = concurrent.futures.ThreadPoolExecutor(max_workers=50)
+        executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
 
         def blocking_lookup(filename: Path, prover: DiskProver) -> Optional[List]:
             # Uses the DiskProver object to lookup qualities. This is a blocking call,

src/harvester.py

@@ -99,11 +99,11 @@ def unflatten_properties(config: Dict):
 
 
 def add_property(d: Dict, partial_key: str, value: Any):
-    key_1, key_2 = partial_key.split(".")
+    key_1, key_2 = partial_key.split(".", maxsplit=1)
     if key_1 not in d:
         d[key_1] = {}
     if "." in key_2:
-        add_property(d, key_2, value)
+        add_property(d[key_1], key_2, value)
     else:
         d[key_1][key_2] = value