WIP move to ruimarinho images + other changes

.gitignore

@@ -3,4 +3,5 @@ __pycache__
 .venv
 warnet.egg-info
 .python-version
-.env
+.env
+src/templates/Dockerfile_[0-9]*

README.md

@@ -111,33 +111,43 @@ pip install -e .
 
 ## Running
 
-Warnet runs a daemon called `warnetd` which can be used to manage multiple warnets.
-`warnetd` will by default log to a file `$XDG_STATE_HOME/warnet/warnet.log` if the `$XDG_STATE_HOME` environment variable is set, otherwise it will use `$HOME/.local/state/warnet/warnet.log`.
+Warnet runs a daemon which can be used to manage multiple warnets.
+`warnet` will by default log to a file `$XDG_STATE_HOME/warnet/warnet.log` if the `$XDG_STATE_HOME` environment variable is set, otherwise it will use `$HOME/.warnet/warnet.log`.
 
-To start `warnetd` with your venv activated simply run:
+To start `warnet` in the foreground with your venv activated simply run:
 
 ```bash
-warnetd
+warnet
 ```
 
 > [!NOTE]
-> `warnetd` also accepts a `--no-debug` option which prevents daemonization
+> `warnetd` also accepts a `--daemon` option which runs the process in the background.
 
-Once `warnetd` is running it can be interacted with using the cli tool `warnet`.
-Run `warnet --help` to see a list of possible commands.
+Run `warnet --help` to see a list of options.
 
-All `warnet` commands accept a `--network` option, which allows you to specify the warnet you want to control.
+Once `warnet` is running it can be interacted with using the cli tool `warcli`.
+All `warcli` commands accept a `--network` option, which allows you to specify the warnet you want to control.
 This is set by default to `--network="warnet"` to simplify default operation.
 
-To start an example warnet, with your venv active, run the following command to use the default graph and network:
+To start an example warnet, with your venv active and the server running, run the following command to use the default graph and network:
 
 ```bash
-warnet start
+warcli start
+```
+
+To see available commands use:
+
+```bash
+# All commands help
+warcli help
+
+# Sub-command help
+warcli help networks
 ```
 
 Each container is a node as described in the graph, along with various data exporters and a demo grafana dashboard.
 
-The commands listed in `warnet --help` can then be used to control and query the nodes.
+The commands listed in `warcli help` can then be used to control and query the nodes.
 
 ###  Run scenarios on a network
 
@@ -150,31 +160,31 @@ See `/src/scenarios` for examples of how these can be written.
 To see available scenarios (loaded from the default directory):
 
 ```bash
-warnet list
+warcli scenarios list
 ```
 
-Once a scenarios is selected it can be run with `warnet run <scenario_name> [--network=warnet]`, e.g.:
+Once a scenarios is selected it can be run with `warcli scenarios run <scenario_name> [--network=warnet]`, e.g.:
 
 ```bash
 # Command one node to generate a wallet and fill 100 blocks with 100 txs each
-warnet run tx-flood.py
+warcli scenarios run tx-flood.py
 ```
 
-This will run the run the scenario in the background until it exits, or is killed by the user.
+This will run the run the scenario in the background until it exits or is killed by the user.
 
 ### Stopping
 
-Currently the warnet can be stopped, or stopped and removed, but **not** stopped, persisted and restarted.
+Currently the warnet can be stopped, but **not** stopped, persisted and continued.
 Persisting the warnet during a stoppage is WIP.
 
-To stop the warnet, or remove it (which first stops, then deletes the containers):
+To stop the warnet server:
 
 ```bash
 # stop but retain containers
-warnet stop
+warcli network down
 
-# stop and erase containers
-warnet wipe
+# stop warnetd
+warcli stop
 ```
 
 ## Remote / Cloud Deployment

pyproject.toml

@@ -13,22 +13,24 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
+    "black==23.7.0",
+    "click==8.1.7",
     "docker==6.1.3",
-    "networkx==3.1",
-    "PyYAML==6.0.1",
-    "typer[all]==0.9.0",
+    "flask==2.3.3",
+    "Flask-JSONRPC==2.2.2",
+    "gunicorn==21.2.0",
     "jsonschema",
     "jsonrpcserver==5.0.3",
     "jsonrpcclient==4.0.0",
-    "gunicorn==21.2.0",
-    "flask==2.3.3",
-    "Flask-JSONRPC==2.2.2",
+    "networkx==3.1",
+    "rich==13.5.2",
+    "PyYAML==6.0.1",
 ]
 dynamic = ["version"]
 
 [project.scripts]
-warnetd = "warnet.warnetd:run_gunicorn"
-warnet = "warnet.cli:cli"
+warnet = "warnet.warnetd:run_gunicorn"
+warcli = "warnet.cli.main:cli"
 
 [tool.black]
 line-length = 88

src/scenarios/tx_flood.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-
 from warnet.test_framework_bridge import WarnetTestFramework
 from scenarios.utils import ensure_miner
 

src/services/base_service.py

@@ -0,0 +1,11 @@
+from pathlib import Path
+
+
+class BaseService:
+    def __init__(self, docker_network, config_dir=Path()):
+        self.docker_network = docker_network
+        self.config_dir = config_dir
+        self.service = {}
+
+    def get_service(self):
+        return self.service

src/services/dns_seed.py

@@ -0,0 +1,27 @@
+from .base_service import BaseService
+import shutil
+
+
+PORT = 15353
+DNS_SEED_NAME = "dns-seed"
+ZONE_FILE_NAME = "dns-seed.zone"
+NAMED_CONF_NAME = "named.conf.local"
+
+
+class DnsSeed(BaseService):
+    def __init__(self, docker_network, templates, config_dir):
+        super().__init__(docker_network)
+        self.docker_network = docker_network
+        self.templates = templates
+        self.service = {
+            "container_name": f"{self.docker_network}_{DNS_SEED_NAME}",
+            "ports": [f"{PORT}:53/udp", f"{PORT}:53/tcp"],
+            "build": {
+                "context": ".",
+                "dockerfile": str(self.templates / "Dockerfile_bind9"),
+            },
+            "networks": [self.docker_network],
+        }
+        # Copy files for dockerfile
+        shutil.copy(str(self.templates / ZONE_FILE_NAME), config_dir)
+        shutil.copy(str(self.templates / NAMED_CONF_NAME), config_dir)

src/services/fluentd.py

@@ -0,0 +1,24 @@
+from .base_service import BaseService
+
+FLUENT_IP = "100.102.108.117"
+FLUENT_CONF = "fluent.conf"
+
+class Fluentd(BaseService):
+    PORT = 24224
+
+    def __init__(self, docker_network, config_dir):
+        super().__init__(docker_network, config_dir)
+        self.service = {
+            "image": "fluent/fluentd:v1.16-debian-1", # Debian version is recommended officially since it has jemalloc support.
+            "container_name": f"{self.docker_network}_fluentd",
+            "ports": [f"{self.PORT}:{self.PORT}"],
+            "volumes": [
+                f"{self.config_dir / FLUENT_CONF}:/fluentd/etc/{FLUENT_CONF}"
+            ],
+            "command": ["/bin/sh", "-c", f"sleep 10 && fluentd -c /fluentd/etc/{FLUENT_CONF}"],
+            "networks": {
+                self.docker_network: {
+                    "ipv4_address": f"{FLUENT_IP}",
+                }
+            },
+        }

src/services/fork_observer.py

@@ -0,0 +1,16 @@
+from .base_service import BaseService
+
+PORT = 12323
+
+
+class ForkObserver(BaseService):
+    def __init__(self, docker_network, fork_observer_config):
+        super().__init__(docker_network)
+        self.fork_observer_config = fork_observer_config
+        self.service = {
+            "image": "b10c/fork-observer:latest",
+            "container_name": f"{self.docker_network}_fork-observer",
+            "ports": [f"{PORT}:2323"],
+            "volumes": [f"{self.fork_observer_config}:/app/config.toml"],
+            "networks": [self.docker_network],
+        }

src/services/grafana.py

@@ -0,0 +1,15 @@
+from .base_service import BaseService
+
+PORT = 3000
+
+
+class Grafana(BaseService):
+    def __init__(self, docker_network):
+        super().__init__(docker_network)
+        self.service = {
+            "image": "grafana/grafana:latest",
+            "container_name": f"{self.docker_network}_grafana",
+            "ports": [f"3000:{PORT}"],
+            "volumes": ["grafana-storage:/var/lib/grafana"],
+            "networks": [self.docker_network],
+        }

src/services/node_exporter.py

@@ -0,0 +1,13 @@
+from .base_service import BaseService
+
+
+class NodeExporter(BaseService):
+    def __init__(self, docker_network):
+        super().__init__(docker_network)
+        self.service = {
+            "image": "prom/node-exporter:latest",
+            "container_name": f"{self.docker_network}_node-exporter",
+            "volumes": ["/proc:/host/proc:ro", "/sys:/host/sys:ro", "/:/rootfs:ro"],
+            "command": ["--path.procfs=/host/proc", "--path.sysfs=/host/sys"],
+            "networks": [self.docker_network],
+        }

src/services/prometheus.py

@@ -0,0 +1,19 @@
+import docker
+from .base_service import BaseService
+
+PORT = 9090
+
+
+class Prometheus(BaseService):
+    def __init__(self, docker_network, config_dir):
+        super().__init__(docker_network, config_dir)
+        self.service = {
+            "image": "prom/prometheus:latest",
+            "container_name": f"{self.docker_network}_prometheus",
+            "ports": [f"{PORT}:9090"],
+            "volumes": [
+                f"{self.config_dir / 'prometheus.yml'}:/etc/prometheus/prometheus.yml"
+            ],
+            "command": ["--config.file=/etc/prometheus/prometheus.yml"],
+            "networks": [self.docker_network],
+        }

src/services/tor.py

@@ -0,0 +1,22 @@
+from .base_service import BaseService
+
+DOCKERFILE = "Dockerfile_tor_da"
+DIRECTORY_AUTHORITY_IP = "100.20.15.18"
+
+
+class Tor(BaseService):
+    def __init__(self, docker_network, templates):
+        super().__init__(docker_network)
+        self.templates = templates
+        self.service = {
+            "build": {
+                "context": str(self.templates),
+                "dockerfile": DOCKERFILE,
+            },
+            "container_name": f"{self.docker_network}_tor",
+            "networks": {
+                self.docker_network: {
+                    "ipv4_address": DIRECTORY_AUTHORITY_IP,
+                }
+            },
+        }