Merge pull request #528 from josibake/refactor-config

Refactor hardcoded strings into a config / constants file

Author: pinheadmz

src/warnet/admin.py

@@ -4,6 +4,7 @@
 import click
 from rich import print as richprint
 
+from .constants import NETWORK_DIR
 from .namespaces import copy_namespaces_defaults, namespaces
 from .network import copy_network_defaults
 
@@ -17,22 +18,6 @@ def admin():
 admin.add_command(namespaces)
 
 
-@admin.command()
-@click.argument("directory", type=Path)
-def create(directory):
-    """Create a new warnet project in the specified directory"""
-    if os.path.exists(directory):
-        richprint(f"[red]Error: Directory {directory} already exists[/red]")
-        return
-
-    copy_network_defaults(directory)
-    copy_namespaces_defaults(directory)
-    richprint(
-        f"[green]Copied network and namespace example files to {directory / 'networks'}[/green]"
-    )
-    richprint(f"[green]Created warnet project structure in {directory}[/green]")
-
-
 @admin.command()
 def init():
     """Initialize a warnet project in the current directory"""
@@ -45,6 +30,6 @@ def init():
     copy_network_defaults(Path(current_dir))
     copy_namespaces_defaults(Path(current_dir))
     richprint(
-        f"[green]Copied network and namespace example files to {Path(current_dir) / 'networks'}[/green]"
+        f"[green]Copied network and namespace example files to {Path(current_dir) / NETWORK_DIR.name}[/green]"
     )
     richprint(f"[green]Created warnet project structure in {current_dir}[/green]")

src/warnet/constants.py

@@ -0,0 +1,92 @@
+import os
+from importlib.resources import files
+from pathlib import Path
+
+# Constants used throughout the project
+# Storing as constants for now but we might want a more sophisticated config management
+# at some point.
+SUPPORTED_TAGS = ["27.0", "26.0", "25.1", "24.2", "23.2", "22.2"]
+DEFAULT_TAG = SUPPORTED_TAGS[0]
+WEIGHTED_TAGS = [
+    tag for index, tag in enumerate(reversed(SUPPORTED_TAGS)) for _ in range(index + 1)
+]
+
+DEFAULT_NAMESPACE = "warnet"
+HELM_COMMAND = "helm upgrade --install --create-namespace"
+
+# Directories and files for non-python assets, e.g., helm charts, example scenarios, default configs
+SRC_DIR = files("warnet")
+RESOURCES_DIR = files("resources")
+NETWORK_DIR = RESOURCES_DIR.joinpath("networks")
+SCENARIOS_DIR = RESOURCES_DIR.joinpath("scenarios")
+CHARTS_DIR = RESOURCES_DIR.joinpath("charts")
+MANIFESTS_DIR = RESOURCES_DIR.joinpath("manifests")
+NETWORK_FILE = "network.yaml"
+DEFAULTS_FILE = "node-defaults.yaml"
+NAMESPACES_FILE = "namespaces.yaml"
+
+# Helm charts
+BITCOIN_CHART_LOCATION = str(CHARTS_DIR.joinpath("bitcoincore"))
+FORK_OBSERVER_CHART = str(CHARTS_DIR.joinpath("fork-observer"))
+NAMESPACES_CHART_LOCATION = CHARTS_DIR.joinpath("namespaces")
+DEFAULT_NETWORK = Path("6_node_bitcoin")
+DEFAULT_NAMESPACES = Path("two_namespaces_two_users")
+
+# Kubeconfig related stuffs
+KUBECONFIG = os.environ.get("KUBECONFIG", os.path.expanduser("~/.kube/config"))
+
+# TODO: all of this logging stuff should be a helm chart
+LOGGING_CONFIG = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "simple": {
+            "format": "%(asctime)s | %(levelname)-7s | %(name)-8s | %(message)s",
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
+        "detailed": {
+            "format": "%(asctime)s | %(levelname)-7s | [%(module)21s:%(lineno)4d] | %(message)s",
+            "datefmt": "%Y-%m-%d %H:%M:%S",
+        },
+    },
+    "handlers": {
+        "stdout": {
+            "class": "logging.StreamHandler",
+            "level": "DEBUG",
+            "formatter": "simple",
+            "stream": "ext://sys.stdout",
+        },
+        "stderr": {
+            "class": "logging.StreamHandler",
+            "level": "WARNING",
+            "formatter": "simple",
+            "stream": "ext://sys.stderr",
+        },
+        "file": {
+            "class": "logging.handlers.RotatingFileHandler",
+            "level": "DEBUG",
+            "formatter": "detailed",
+            "filename": "warnet.log",
+            "maxBytes": 16000000,
+            "backupCount": 3,
+        },
+    },
+    "loggers": {
+        "root": {"level": "DEBUG", "handlers": ["stdout", "stderr", "file"]},
+        "urllib3.connectionpool": {"level": "WARNING", "propagate": 1},
+        "kubernetes.client.rest": {"level": "WARNING", "propagate": 1},
+        "werkzeug": {"level": "WARNING", "propagate": 1},
+    },
+}
+
+# Helm commands for logging setup
+# TODO: also lots of hardcode stuff in these helm commands, will need to fix this when moving to helm charts
+LOGGING_HELM_COMMANDS = [
+    "helm repo add grafana https://grafana.github.io/helm-charts",
+    "helm repo add prometheus-community https://prometheus-community.github.io/helm-charts",
+    "helm repo update",
+    f"helm upgrade --install --namespace warnet-logging --create-namespace --values {MANIFESTS_DIR}/loki_values.yaml loki grafana/loki --version 5.47.2",
+    "helm upgrade --install --namespace warnet-logging promtail grafana/promtail",
+    "helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false",
+    f"helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values {MANIFESTS_DIR}/grafana_values.yaml",
+]

src/warnet/deploy.py

@@ -4,32 +4,17 @@
 import click
 import yaml
 
-from .k8s import get_default_namespace
-from .namespaces import (
-    BITCOIN_CHART_LOCATION as NAMESPACES_CHART_LOCATION,
-)
-from .namespaces import (
-    DEFAULTS_FILE as NAMESPACES_DEFAULTS_FILE,
-)
-from .namespaces import (
-    NAMESPACES_FILE,
-)
-from .network import (
-    BITCOIN_CHART_LOCATION as NETWORK_CHART_LOCATION,
-)
-from .network import (
-    DEFAULTS_FILE as NETWORK_DEFAULTS_FILE,
-)
-
-# Import necessary functions and variables from network.py and namespaces.py
-from .network import (
+from .constants import (
+    BITCOIN_CHART_LOCATION,
+    DEFAULTS_FILE,
     FORK_OBSERVER_CHART,
+    HELM_COMMAND,
+    NAMESPACES_FILE,
     NETWORK_FILE,
 )
+from .k8s import get_default_namespace
 from .process import stream_command
 
-HELM_COMMAND = "helm upgrade --install --create-namespace"
-
 
 def validate_directory(ctx, param, value):
     directory = Path(value)
@@ -82,7 +67,6 @@ def deploy_fork_observer(directory: Path, debug: bool):
     override_string = ""
 
     # Add an entry for each node in the graph
-    # TODO: should this be moved into a chart, and only have substituted name and rpc_host values
     for i, node in enumerate(network_file["nodes"]):
         node_name = node.get("name")
         node_config = f"""
@@ -97,7 +81,6 @@ def deploy_fork_observer(directory: Path, debug: bool):
 """
 
         override_string += node_config
-        # End loop
 
     # Create yaml string using multi-line string format
     override_string = override_string.strip()
@@ -119,7 +102,7 @@ def deploy_fork_observer(directory: Path, debug: bool):
 
 def deploy_network(directory: Path, debug: bool = False):
     network_file_path = directory / NETWORK_FILE
-    defaults_file_path = directory / NETWORK_DEFAULTS_FILE
+    defaults_file_path = directory / DEFAULTS_FILE
 
     with network_file_path.open() as f:
         network_file = yaml.safe_load(f)
@@ -133,7 +116,7 @@ def deploy_network(directory: Path, debug: bool = False):
             node_name = node.get("name")
             node_config_override = {k: v for k, v in node.items() if k != "name"}
 
-            cmd = f"{HELM_COMMAND} {node_name} {NETWORK_CHART_LOCATION} --namespace {namespace} -f {defaults_file_path}"
+            cmd = f"{HELM_COMMAND} {node_name} {BITCOIN_CHART_LOCATION} --namespace {namespace} -f {defaults_file_path}"
             if debug:
                 cmd += " --debug"
 
@@ -158,7 +141,7 @@ def deploy_network(directory: Path, debug: bool = False):
 
 def deploy_namespaces(directory: Path):
     namespaces_file_path = directory / NAMESPACES_FILE
-    defaults_file_path = directory / NAMESPACES_DEFAULTS_FILE
+    defaults_file_path = directory / DEFAULTS_FILE
 
     with namespaces_file_path.open() as f:
         namespaces_file = yaml.safe_load(f)
@@ -178,7 +161,9 @@ def deploy_namespaces(directory: Path):
             namespace_name = namespace.get("name")
             namespace_config_override = {k: v for k, v in namespace.items() if k != "name"}
 
-            cmd = f"{HELM_COMMAND} {namespace_name} {NAMESPACES_CHART_LOCATION} -f {defaults_file_path}"
+            cmd = (
+                f"{HELM_COMMAND} {namespace_name} {BITCOIN_CHART_LOCATION} -f {defaults_file_path}"
+            )
 
             if namespace_config_override:
                 with tempfile.NamedTemporaryFile(

src/warnet/k8s.py

@@ -7,18 +7,17 @@
 from kubernetes.client.models import CoreV1Event, V1PodList
 from kubernetes.dynamic import DynamicClient
 
+from .constants import DEFAULT_NAMESPACE, KUBECONFIG
 from .process import run_command, stream_command
 
-DEFAULT_NAMESPACE = "warnet"
-
 
 def get_static_client() -> CoreV1Event:
-    config.load_kube_config()
+    config.load_kube_config(config_file=KUBECONFIG)
     return client.CoreV1Api()
 
 
 def get_dynamic_client() -> DynamicClient:
-    config.load_kube_config()
+    config.load_kube_config(config_file=KUBECONFIG)
     return DynamicClient(client.ApiClient())
 
 

src/warnet/main.py

@@ -6,7 +6,6 @@
 import sys
 from dataclasses import dataclass
 from enum import Enum, auto
-from importlib.resources import files
 from pathlib import Path
 from typing import Callable
 
@@ -15,6 +14,14 @@
 import yaml
 from inquirer.themes import GreenPassion
 
+from warnet.constants import (
+    DEFAULT_TAG,
+    DEFAULTS_FILE,
+    NETWORK_DIR,
+    NETWORK_FILE,
+    SRC_DIR,
+    SUPPORTED_TAGS,
+)
 from warnet.k8s import get_default_namespace
 from warnet.process import run_command, stream_command
 
@@ -26,9 +33,8 @@
 from .image import image
 from .network import copy_network_defaults, copy_scenario_defaults
 from .status import status as status_command
-from .util import DEFAULT_TAG, SUPPORTED_TAGS
 
-QUICK_START_PATH = files("resources.scripts").joinpath("quick_start.sh")
+QUICK_START_PATH = SRC_DIR.joinpath("resources", "scripts", "quick_start.sh")
 
 
 @click.group()
@@ -592,10 +598,6 @@ def logs(pod_name: str, follow: bool):
         pass  # cancelled by user
 
 
-if __name__ == "__main__":
-    cli()
-
-
 def custom_graph(
     num_nodes: int,
     num_connections: int,
@@ -639,17 +641,19 @@ def custom_graph(
         "configQueryInterval": fork_obs_query_interval,
     }
 
-    with open(os.path.join(datadir, "network.yaml"), "w") as f:
+    with open(os.path.join(datadir, NETWORK_FILE), "w") as f:
         yaml.dump(network_yaml_data, f, default_flow_style=False)
 
     # Generate node-defaults.yaml
-    default_yaml_path = files("resources.networks").joinpath("node-defaults.yaml")
+    default_yaml_path = NETWORK_DIR.joinpath(DEFAULTS_FILE)
     with open(str(default_yaml_path)) as f:
         defaults_yaml_content = f.read()
 
-    with open(os.path.join(datadir, "node-defaults.yaml"), "w") as f:
+    with open(os.path.join(datadir, DEFAULTS_FILE), "w") as f:
         f.write(defaults_yaml_content)
 
-    click.echo(
-        f"Project '{datadir}' has been created with 'network.yaml' and 'node-defaults.yaml'."
-    )
+    click.echo(f"Project '{datadir}' has been created with '{NETWORK_FILE}' and '{DEFAULTS_FILE}'.")
+
+
+if __name__ == "__main__":
+    cli()

src/warnet/namespaces.py

@@ -1,31 +1,24 @@
 import shutil
-from importlib.resources import files
 from pathlib import Path
 
 import click
 
+from .constants import (
+    DEFAULT_NAMESPACES,
+    DEFAULTS_FILE,
+    NAMESPACES_FILE,
+    NETWORK_DIR,
+)
 from .process import run_command, stream_command
 
-WARNET_NAMESPACES_DIR = files("resources").joinpath("namespaces")
-NAMESPACES_DIR = Path("namespaces")
-DEFAULT_NAMESPACES = Path("two_namespaces_two_users")
-NAMESPACES_FILE = "namespaces.yaml"
-DEFAULTS_FILE = "namespace-defaults.yaml"
-HELM_COMMAND = "helm upgrade --install"
-BITCOIN_CHART_LOCATION = Path(str(files("resources.charts").joinpath("namespaces")))
-
 
 def copy_namespaces_defaults(directory: Path):
     """Create the project structure for a warnet project"""
-    (directory / NAMESPACES_DIR / DEFAULT_NAMESPACES).mkdir(parents=True, exist_ok=True)
-    target_namespaces_defaults = directory / NAMESPACES_DIR / DEFAULT_NAMESPACES / DEFAULTS_FILE
-    target_namespaces_example = directory / NAMESPACES_DIR / DEFAULT_NAMESPACES / NAMESPACES_FILE
-    shutil.copy2(
-        WARNET_NAMESPACES_DIR / DEFAULT_NAMESPACES / DEFAULTS_FILE, target_namespaces_defaults
-    )
-    shutil.copy2(
-        WARNET_NAMESPACES_DIR / DEFAULT_NAMESPACES / NAMESPACES_FILE, target_namespaces_example
-    )
+    (directory / NETWORK_DIR / DEFAULT_NAMESPACES).mkdir(parents=True, exist_ok=True)
+    target_namespaces_defaults = directory / NETWORK_DIR / DEFAULT_NAMESPACES / DEFAULTS_FILE
+    target_namespaces_example = directory / NETWORK_DIR / DEFAULT_NAMESPACES / NAMESPACES_FILE
+    shutil.copy2(NETWORK_DIR / DEFAULT_NAMESPACES / DEFAULTS_FILE, target_namespaces_defaults)
+    shutil.copy2(NETWORK_DIR / DEFAULT_NAMESPACES / NAMESPACES_FILE, target_namespaces_example)
 
 
 @click.group(name="namespaces")