address Max's feedback

- Fix multithreading on macos
- Create a separate network file to test the plugin system
- Create useful messages in the console when doing plugin operations
- Avoid reading the yaml file; instead pass along only those values needed

Author: mplsgrant

resources/plugins/simln/plugin.py

@@ -5,12 +5,11 @@
 from pathlib import Path
 
 import click
-import yaml
 from kubernetes.stream import stream
 
 # When we want to select pods based on their role in Warnet, we use "mission" tags. The "mission"
 # tag for "lightning" nodes is stored in LIGHTNING_MISSION.
-from warnet.constants import LIGHTNING_MISSION, HookValue
+from warnet.constants import LIGHTNING_MISSION, PLUGIN_ANNEX, AnnexMember, HookValue, WarnetContent
 from warnet.k8s import (
     download,
     get_default_namespace,
@@ -61,39 +60,35 @@ def simln(ctx):
 # value and a variable number of arguments which is used by, for example, preNode and postNode to
 # pass along node names.
 @simln.command()
-@click.argument("network_file_path", type=str)
-@click.argument("hook_value", type=str)
-@click.argument("namespace", type=str)
-@click.argument("nargs", nargs=-1)
+@click.argument("plugin_content", type=str)
+@click.argument("warnet_content", type=str)
 @click.pass_context
-def entrypoint(ctx, network_file_path: str, hook_value: str, namespace: str, nargs):
+def entrypoint(ctx, plugin_content: str, warnet_content: str):
     """Plugin entrypoint"""
+    plugin_content: dict = json.loads(plugin_content)
+    warnet_content: dict = json.loads(warnet_content)
+
+    hook_value = warnet_content.get(WarnetContent.HOOK_VALUE.value)
+
     assert hook_value in {
         item.value for item in HookValue
     }, f"{hook_value} is not a valid HookValue"
 
-    network_file_path = Path(network_file_path)
-
-    with network_file_path.open() as f:
-        network_file = yaml.safe_load(f) or {}
-        if not isinstance(network_file, dict):
-            raise ValueError(f"Invalid network file structure: {network_file_path}")
-
-    plugins_section = network_file.get("plugins", {})
-    hook_section = plugins_section.get(hook_value, {})
+    if warnet_content.get(PLUGIN_ANNEX):
+        for annex_member in [annex_item for annex_item in warnet_content.get(PLUGIN_ANNEX)]:
+            assert annex_member in {
+                item.value for item in AnnexMember
+            }, f"{annex_member} is not a valid AnnexMember"
 
-    plugin_name = Path(__file__).resolve().parent.stem
-    plugin_data = hook_section.get(plugin_name)
-    if not plugin_data:
-        raise PluginError(f"Could not find {plugin_name} in {network_file_path}")
+    warnet_content[WarnetContent.HOOK_VALUE.value] = HookValue(hook_value)
 
-    _entrypoint(ctx, plugin_data, HookValue(hook_value), namespace, nargs)
+    _entrypoint(ctx, plugin_content, warnet_content)
 
 
-def _entrypoint(ctx, plugin_data: dict, hook_value: HookValue, namespace: str, nargs):
+def _entrypoint(ctx, plugin_content: dict, warnet_content: dict):
     """Called by entrypoint"""
     # write your plugin startup commands here
-    activity = plugin_data.get("activity")
+    activity = plugin_content.get("activity")
     activity = json.loads(activity)
     print(activity)
     _launch_activity(activity, ctx.obj.get(PLUGIN_DIR_TAG))

src/warnet/constants.py

@@ -36,6 +36,19 @@ class HookValue(Enum):
     POST_NETWORK = "postNetwork"
 
 
+class WarnetContent(Enum):
+    HOOK_VALUE = "hook_value"
+    NAMESPACE = "namespace"
+    ANNEX = "annex"
+
+
+class AnnexMember(Enum):
+    NODE_NAME = "node_name"
+
+
+PLUGIN_ANNEX = "annex"
+
+
 # Directories and files for non-python assets, e.g., helm charts, example scenarios, default configs
 SRC_DIR = files("warnet")
 RESOURCES_DIR = files("resources")

src/warnet/deploy.py

@@ -1,3 +1,4 @@
+import json
 import subprocess
 import sys
 import tempfile
@@ -22,9 +23,12 @@
     NAMESPACES_CHART_LOCATION,
     NAMESPACES_FILE,
     NETWORK_FILE,
+    PLUGIN_ANNEX,
     SCENARIOS_DIR,
     WARGAMES_NAMESPACE_PREFIX,
+    AnnexMember,
     HookValue,
+    WarnetContent,
 )
 from .control import _run
 from .k8s import (
@@ -132,7 +136,7 @@ def _deploy(directory, debug, namespace, to_all_users):
         )
 
 
-def run_plugins(directory, hook_value: HookValue, namespace, *args):
+def run_plugins(directory, hook_value: HookValue, namespace, annex: Optional[dict] = None):
     """Run the plugin commands within a given hook value"""
 
     network_file_path = directory / NETWORK_FILE
@@ -146,29 +150,44 @@ def run_plugins(directory, hook_value: HookValue, namespace, *args):
 
     plugins_section = network_file.get("plugins", {})
     hook_section = plugins_section.get(hook_value.value, {})
-    for plugin_cmd in hook_section.items():
-        match plugin_cmd:
+    for plugin_name, plugin_content in hook_section.items():
+        match (plugin_name, plugin_content):
             case (str(), dict()):
                 try:
-                    entrypoint_path = Path(plugin_cmd[1].get("entrypoint"))
+                    entrypoint_path = Path(plugin_content.get("entrypoint"))
                 except Exception as err:
                     raise SyntaxError("Each plugin must have an 'entrypoint'") from err
 
-                cmd = f"{network_file_path.parent / entrypoint_path / Path('plugin.py')} entrypoint {network_file_path} {hook_value.value} {namespace} {' '.join(map(str, args))}"
+                warnet_content = {
+                    WarnetContent.HOOK_VALUE.value: hook_value.value,
+                    WarnetContent.NAMESPACE.value: namespace,
+                    PLUGIN_ANNEX: annex,
+                }
+
+                cmd = (
+                    f"{network_file_path.parent / entrypoint_path / Path('plugin.py')} entrypoint "
+                    f"'{json.dumps(plugin_content)}' '{json.dumps(warnet_content)}'"
+                )
+                print(
+                    f"Queuing {hook_value.value} plugin command: {plugin_name} with {plugin_content}"
+                )
+
                 process = Process(target=run_command, args=(cmd,))
                 processes.append(process)
 
             case _:
                 print(
-                    f"The following plugin command does not match known plugin command structures: {plugin_cmd}"
+                    f"The following plugin command does not match known plugin command structures: {plugin_name} {plugin_content}"
                 )
                 sys.exit(1)
 
+    print(f"Starting {hook_value.value} plugins")
     for process in processes:
         process.start()
 
     for process in processes:
         process.join()
+    print(f"Completed {hook_value.value} plugins")
 
 
 def check_logging_required(directory: Path):
@@ -385,13 +404,20 @@ def deploy_single_node(node, directory: Path, debug: bool, namespace: str):
                 temp_override_file_path = Path(temp_file.name)
             cmd = f"{cmd} -f {temp_override_file_path}"
 
-        run_plugins(directory, HookValue.PRE_NODE, namespace, node_name)
+        run_plugins(
+            directory, HookValue.PRE_NODE, namespace, annex={AnnexMember.NODE_NAME.value: node_name}
+        )
 
         if not stream_command(cmd):
             click.echo(f"Failed to run Helm command: {cmd}")
             return
 
-        run_plugins(directory, HookValue.POST_NODE, namespace, node_name)
+        run_plugins(
+            directory,
+            HookValue.POST_NODE,
+            namespace,
+            annex={AnnexMember.NODE_NAME.value: node_name},
+        )
 
     except Exception as e:
         click.echo(f"Error: {e}")

test/data/ln/network.yaml

@@ -52,37 +52,3 @@ nodes:
       - tank-0000
     ln:
       lnd: true
-
-plugins:
-  preDeploy:
-    hello:
-      entrypoint: "../plugins/hello"
-      podName: "hello-pre-deploy"
-      helloTo: "preDeploy!"
-  postDeploy:
-    simln:
-      entrypoint: "../../../resources/plugins/simln"
-      activity: '[{"source": "tank-0003-ln", "destination": "tank-0005-ln", "interval_secs": 1, "amount_msat": 2000}]'
-    hello:
-      entrypoint: "../plugins/hello"
-      podName: "hello-post-deploy"
-      helloTo: "postDeploy!"
-  preNode:
-    hello:
-      entrypoint: "../plugins/hello"
-      helloTo: "preNode!"
-  postNode:
-    hello:
-      entrypoint: "../plugins/hello"
-      helloTo: "postNode!"
-  preNetwork:
-    hello:
-      entrypoint: "../plugins/hello"
-      helloTo: "preNetwork!"
-      podName: "hello-pre-network"
-  postNetwork:
-    hello:
-      entrypoint: "../plugins/hello"
-      helloTo: "postNetwork!"
-      podName: "hello-post-network"
-

test/data/plugins/hello/plugin.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
+import json
 import logging
 from pathlib import Path
 from typing import Optional
 
 import click
-import yaml
 from kubernetes.stream import stream
 
-from warnet.constants import HookValue
+from warnet.constants import PLUGIN_ANNEX, AnnexMember, HookValue, WarnetContent
 from warnet.k8s import (
     get_default_namespace,
     get_static_client,
@@ -45,59 +45,59 @@ def hello(ctx):
 
 
 @hello.command()
-@click.argument("network_file_path", type=str)
-@click.argument("hook_value", type=str)
-@click.argument("namespace", type=str)
-@click.argument("nargs", nargs=-1)
+@click.argument("plugin_content", type=str)
+@click.argument("warnet_content", type=str)
 @click.pass_context
-def entrypoint(ctx, network_file_path: str, hook_value: str, namespace: str, nargs):
+def entrypoint(ctx, plugin_content: str, warnet_content: str):
     """Plugin entrypoint"""
+    plugin_content: dict = json.loads(plugin_content)
+    warnet_content: dict = json.loads(warnet_content)
+
+    hook_value = warnet_content.get(WarnetContent.HOOK_VALUE.value)
+
     assert hook_value in {
         item.value for item in HookValue
     }, f"{hook_value} is not a valid HookValue"
 
-    network_file_path = Path(network_file_path)
-
-    with network_file_path.open() as f:
-        network_file = yaml.safe_load(f) or {}
-        if not isinstance(network_file, dict):
-            raise ValueError(f"Invalid network file structure: {network_file_path}")
+    if warnet_content.get(PLUGIN_ANNEX):
+        for annex_member in [annex_item for annex_item in warnet_content.get(PLUGIN_ANNEX)]:
+            assert annex_member in {
+                item.value for item in AnnexMember
+            }, f"{annex_member} is not a valid AnnexMember"
 
-    plugins_section = network_file.get("plugins", {})
-    hook_section = plugins_section.get(hook_value, {})
+    warnet_content[WarnetContent.HOOK_VALUE.value] = HookValue(hook_value)
 
-    plugin_name = Path(__file__).resolve().parent.stem
-    plugin_data = hook_section.get(plugin_name)
-    if not plugin_data:
-        raise PluginError(f"Could not find {plugin_name} in {network_file_path}")
+    _entrypoint(ctx, plugin_content, warnet_content)
 
-    _entrypoint(ctx, plugin_data, HookValue(hook_value), namespace, nargs)
 
-
-def _entrypoint(ctx, plugin_data: dict, hook_value: HookValue, namespace: str, nargs):
+def _entrypoint(ctx, plugin_content: dict, warnet_content: dict):
     """Called by entrypoint"""
+    hook_value = warnet_content[WarnetContent.HOOK_VALUE.value]
+
     match hook_value:
         case (
             HookValue.PRE_NETWORK
             | HookValue.POST_NETWORK
             | HookValue.PRE_DEPLOY
             | HookValue.POST_DEPLOY
         ):
-            data = get_data(plugin_data)
+            data = get_data(plugin_content)
             if data:
                 _launch_pod(ctx, install_name=hook_value.value.lower() + "-hello", **data)
             else:
                 _launch_pod(ctx, install_name=hook_value.value.lower() + "-hello")
         case HookValue.PRE_NODE:
-            name = nargs[0] + "-pre-hello-pod"
+            name = warnet_content[PLUGIN_ANNEX][AnnexMember.NODE_NAME.value] + "-pre-hello-pod"
             _launch_pod(ctx, install_name=hook_value.value.lower() + "-" + name, podName=name)
         case HookValue.POST_NODE:
-            name = nargs[0] + "-post-hello-pod"
+            name = warnet_content[PLUGIN_ANNEX][AnnexMember.NODE_NAME.value] + "-post-hello-pod"
             _launch_pod(ctx, install_name=hook_value.value.lower() + "-" + name, podName=name)
 
 
-def get_data(plugin_data: dict) -> Optional[dict]:
-    data = {key: plugin_data.get(key) for key in ("podName", "helloTo") if plugin_data.get(key)}
+def get_data(plugin_content: dict) -> Optional[dict]:
+    data = {
+        key: plugin_content.get(key) for key in ("podName", "helloTo") if plugin_content.get(key)
+    }
     return data or None