set resource requests and limits on tank object

tell k8s up front the min and max CPU, memory we want for a tank.
can be overriden from a graph file if a tank needs more/less.

since we only get 4cpus for minikube in github actions, trim down
the test graphs to run with <= 6 tanks. we dont really need to test
large graphs in CI , anyways.

Author: josibake

.github/workflows/test.yml

@@ -59,6 +59,8 @@ jobs:
       - uses: azure/[email protected]
       - uses: medyagh/setup-minikube@master
         with:
+          cpus: max
+          memory: 10000m
           mount-path: ${{ github.workspace }}:/mnt/src
       - uses: actions/download-artifact@v4
         with:

src/warnet/backend/kubernetes_backend.py

@@ -406,6 +406,10 @@ def create_bitcoind_container(self, tank: Tank) -> client.V1Container:
                 privileged=True,
                 capabilities=client.V1Capabilities(add=["NET_ADMIN", "NET_RAW"]),
             ),
+            resources=client.V1ResourceRequirements(
+                requests=tank.resources["requests"],
+                limits=tank.resources["limits"],
+            ),
         )
         self.log.debug(
             f"Created bitcoind container for tank {tank.index} using {bitcoind_options=:}"

src/warnet/cli/graph.py

@@ -24,11 +24,11 @@ def create(number: int, outfile: Path, version: str, bitcoin_conf: Path, random:
     Create a cycle graph with <number> nodes, and include 7 extra random outbounds per node.
     Returns XML file as string with or without --outfile option
     """
-    graph = create_cycle_graph(number, version, bitcoin_conf, random)
-
+    graph = create_cycle_graph(number, version, bitcoin_conf, random, None)
     if outfile:
         file_path = Path(outfile)
         nx.write_graphml(graph, file_path, named_key_ids=True)
+    validate_graph_schema(graph)
     bio = BytesIO()
     nx.write_graphml(graph, bio, named_key_ids=True)
     xml_data = bio.getvalue()
@@ -52,7 +52,14 @@ def import_json(infile: Path, outfile: Path, cb: str, ln_image: str):
 
     # Start with a connected L1 graph with the right amount of tanks
     graph = create_cycle_graph(
-        len(json_graph["nodes"]), version=DEFAULT_TAG, bitcoin_conf=None, random_version=False
+        len(json_graph["nodes"]),
+        version=DEFAULT_TAG,
+        bitcoin_conf=None,
+        random_version=False,
+        resources={
+            "requests": {"cpu": "100m", "memory": "250Mi"},
+            "limits": {"cpu": "250m", "memory": "500Mi"},
+        },
     )
 
     # Initialize all the tanks with basic LN node configurations

src/warnet/graph_schema.json

@@ -18,64 +18,84 @@
     "properties": {
       "version": {
         "type": "string",
-        "comment": "Bitcoin Core version with an available Warnet tank image on Dockerhub. May also be a GitHub repository with format user/repository:branch to build from source code"},
+        "comment": "Bitcoin Core version with an available Warnet tank image on Dockerhub. May also be a GitHub repository with format user/repository:branch to build from source code"
+      },
       "image": {
         "type": "string",
-        "comment": "Bitcoin Core Warnet tank image on Dockerhub with the format repository/image:tag"},
+        "comment": "Bitcoin Core Warnet tank image on Dockerhub with the format repository/image:tag"
+      },
       "bitcoin_config": {
         "type": "string",
         "default": "",
-        "comment": "A string of Bitcoin Core options in command-line format, e.g. '-debug=net -blocksonly'"},
+        "comment": "A string of Bitcoin Core options in command-line format, e.g. '-debug=net -blocksonly'"
+      },
       "tc_netem": {
         "type": "string",
-        "comment": "A tc-netem command as a string beginning with 'tc qdisc add dev eth0 root netem'"},
+        "comment": "A tc-netem command as a string beginning with 'tc qdisc add dev eth0 root netem'"
+      },
       "exporter": {
         "type": "boolean",
         "default": false,
-        "comment": "Whether to attach a Prometheus data exporter to the tank"},
+        "comment": "Whether to attach a Prometheus data exporter to the tank"
+      },
       "metrics": {
         "type": "string",
-        "comment": "A space-separated string of RPC queries to scrape by Prometheus"},
+        "comment": "A space-separated string of RPC queries to scrape by Prometheus"
+      },
       "collect_logs": {
         "type": "boolean",
-        "default": false,
-        "comment": "Whether to collect Bitcoin Core debug logs with Promtail"},
+        "default": true,
+        "comment": "Whether to collect Bitcoin Core debug logs with Promtail"
+      },
       "build_args": {
         "type": "string",
         "default": "",
-        "comment": "A string of configure options used when building Bitcoin Core from source code, e.g. '--without-gui --disable-tests'"},
+        "comment": "A string of configure options used when building Bitcoin Core from source code, e.g. '--without-gui --disable-tests'"
+      },
       "ln": {
         "type": "string",
-        "comment": "Attach a lightning network node of this implementation (currently only supports 'lnd' or 'cln')"},
+        "comment": "Attach a lightning network node of this implementation (currently only supports 'lnd' or 'cln')"
+      },
       "ln_image": {
         "type": "string",
-        "comment": "Specify a lightning network node image from Dockerhub with the format repository/image:tag"},
+        "comment": "Specify a lightning network node image from Dockerhub with the format repository/image:tag"
+      },
       "ln_cb_image": {
         "type": "string",
-        "comment": "Specify a lnd Circuit Breaker image from Dockerhub with the format repository/image:tag"},
+        "comment": "Specify a lnd Circuit Breaker image from Dockerhub with the format repository/image:tag"
+      },
       "ln_config": {
         "type": "string",
-        "comment": "A string of arguments for the lightning network node in command-line format, e.g. '--protocol.wumbo-channels --bitcoin.timelockdelta=80'"}
+        "comment": "A string of arguments for the lightning network node in command-line format, e.g. '--protocol.wumbo-channels --bitcoin.timelockdelta=80'"
+      },
+      "resources": {
+        "type": "string",
+        "comment": "Kubernetes resource requests and limits for the node",
+        "default": "{'requests':{'cpu':'500m', 'memory':'500Mi'}, 'limits':{'cpu':'1000m', 'memory':'1500Mi'}}"
+      }
     },
-    "additionalProperties": false,
     "oneOf": [
       {"required": ["version"]},
       {"required": ["image"]}
     ],
+   "additionalProperties": false,
     "required": []
   },
   "edge": {
     "type": "object",
     "properties": {
       "channel_open": {
         "type": "string",
-        "comment": "Indicate that this edge is a lightning channel with these arguments passed to lnd openchannel"},
+        "comment": "Indicate that this edge is a lightning channel with these arguments passed to lnd openchannel"
+      },
       "source_policy": {
         "type": "string",
-        "comment": "Update the channel originator policy by passing these arguments passed to lnd updatechanpolicy"},
+        "comment": "Update the channel originator policy by passing these arguments passed to lnd updatechanpolicy"
+      },
       "target_policy": {
         "type": "string",
-        "comment": "Update the channel partner policy by passing these arguments passed to lnd updatechanpolicy"}
+        "comment": "Update the channel partner policy by passing these arguments passed to lnd updatechanpolicy"
+      }
     },
     "additionalProperties": false,
     "required": []

src/warnet/tank.py

@@ -46,6 +46,7 @@ def __init__(self, index: int, warnet):
         self.bitcoin_network = warnet.bitcoin_network
         self.version: str = ""
         self.image: str = ""
+        self.resources: dict = dict()
         self.bitcoin_config = ""
         self.netem = None
         self.exporter = False
@@ -81,7 +82,10 @@ def parse_graph_node(self, node):
             value = node.get(property, specs.get("default"))
             if property == "version":
                 self._parse_version(value)
-            setattr(self, property, value)
+            if property == "resources":
+                setattr(self, property, eval(value))
+            else:
+                setattr(self, property, value)
             graph_properties[property] = value
 
         if self.version and self.image:

src/warnet/utils.py

@@ -380,7 +380,9 @@ def set_execute_permission(file_path):
     os.chmod(file_path, current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
 
 
-def create_cycle_graph(n: int, version: str, bitcoin_conf: str | None, random_version: bool):
+def create_cycle_graph(
+    n: int, version: str, bitcoin_conf: str | None, random_version: bool, resources: dict | None
+):
     try:
         # Use nx.MultiDiGraph() so we get directed edges (source->target)
         # and still allow parallel edges (L1 p2p connections + LN channels)
@@ -421,6 +423,7 @@ def create_cycle_graph(n: int, version: str, bitcoin_conf: str | None, random_ve
                 conf_contents = dump_bitcoin_conf(conf_dict, for_graph=True)
 
     # populate our custom fields
+    graph_schema = load_schema()
     for i, node in enumerate(graph.nodes()):
         if random_version:
             graph.nodes[node]["version"] = random.choice(WEIGHTED_TAGS)
@@ -430,11 +433,11 @@ def create_cycle_graph(n: int, version: str, bitcoin_conf: str | None, random_ve
                 graph.nodes[node]["image"] = f"bitcoindevproject/bitcoin:{version}"
             else:
                 graph.nodes[node]["version"] = version
+        for property, specs in graph_schema["node"]["properties"].items():
+            value = graph.nodes[node].get(property, specs.get("default"))
+            graph.nodes[node][property] = value
+
         graph.nodes[node]["bitcoin_config"] = conf_contents
-        graph.nodes[node]["tc_netem"] = ""
-        graph.nodes[node]["build_args"] = ""
-        graph.nodes[node]["exporter"] = False
-        graph.nodes[node]["collect_logs"] = False
 
     convert_unsupported_attributes(graph)
     return graph

test/dag_connection_test.py

@@ -10,7 +10,7 @@ class DAGConnectionTest(TestBase):
     def __init__(self):
         super().__init__()
         self.graph_file_path = (
-            Path(os.path.dirname(__file__)) / "data" / "ten_semi_unconnected.graphml"
+            Path(os.path.dirname(__file__)) / "data" / "six_semi_unconnected.graphml"
         )
 
     def run_test(self):

test/data/12_node_ring.graphml renamed to test/data/6_node_ring.graphml

@@ -37,30 +37,6 @@
         <data key="bitcoin_config">-uacomment=w4</data>
     </node>
     <node id="5">
-        <data key="version">27.0</data>
-        <data key="bitcoin_config">-uacomment=w5</data>
-    </node>
-    <node id="6">
-        <data key="version">27.0</data>
-        <data key="bitcoin_config">-uacomment=w6</data>
-    </node>
-    <node id="7">
-        <data key="version">27.0</data>
-        <data key="bitcoin_config">-uacomment=w7</data>
-    </node>
-    <node id="8">
-        <data key="version">27.0</data>
-        <data key="bitcoin_config">-uacomment=w8</data>
-    </node>
-    <node id="9">
-        <data key="version">27.0</data>
-        <data key="bitcoin_config">-uacomment=w9</data>
-    </node>
-    <node id="10">
-        <data key="version">27.0</data>
-        <data key="bitcoin_config">-uacomment=w10</data>
-    </node>
-    <node id="11">
         <data key="version">27.0</data>
         <!-- no bitcoin_config to test unused options -->
     </node>
@@ -70,12 +46,6 @@
     <edge id="13" source="2" target="3"></edge>
     <edge id="14" source="3" target="4"></edge>
     <edge id="15" source="4" target="5"></edge>
-    <edge id="16" source="5" target="6"></edge>
-    <edge id="17" source="6" target="7"></edge>
-    <edge id="18" source="7" target="8"></edge>
-    <edge id="19" source="8" target="9"></edge>
-    <edge id="20" source="9" target="10"></edge>
-    <edge id="21" source="10" target="11"></edge>
-    <edge id="21" source="11" target="0"></edge>
+    <edge id="16" source="5" target="0"></edge>
   </graph>
 </graphml>

test/data/permutations.graphml

@@ -64,30 +64,12 @@
       <data key="exporter">False</data>
       <data key="collect_logs">False</data>
     </node>
-    <node id="6">
-      <data key="version">27.0</data>
-      <data key="bitcoin_config" />
-      <data key="tc_netem" />
-      <data key="build_args" />
-      <data key="exporter">False</data>
-      <data key="collect_logs">False</data>
-    </node>
-    <node id="7">
-      <data key="version">27.0</data>
-      <data key="bitcoin_config" />
-      <data key="tc_netem" />
-      <data key="build_args" />
-      <data key="exporter">False</data>
-      <data key="collect_logs">False</data>
-    </node>
     <edge source="0" target="2" id="0" />
     <edge source="1" target="2" id="0" />
     <edge source="1" target="3" id="0" />
     <edge source="2" target="3" id="0" />
     <edge source="2" target="4" id="0" />
     <edge source="3" target="5" id="0" />
     <edge source="5" target="4" id="0" />
-    <edge source="5" target="6" id="0" />
-    <edge source="6" target="7" id="0" />
   </graph>
-</graphml>
+</graphml>

test/data/scenario_connect_dag.py

@@ -20,7 +20,7 @@ class ConnectionType(Enum):
 class ConnectDag(WarnetTestFramework):
     def set_test_params(self):
         # This is just a minimum
-        self.num_nodes = 10
+        self.num_nodes = 6
 
     def add_options(self, parser):
         parser.add_argument(
@@ -63,13 +63,6 @@ def run_test(self):
         self.connect_nodes(2, 4)
         self.connect_nodes(3, 5)
         self.connect_nodes(5, 4)
-        self.connect_nodes(5, 6)
-        self.connect_nodes(6, 7)
-
-        # Nodes 8 & 9 shall come pre-connected. Attempt to connect them anyway to test the handling
-        # of dns node addresses
-        self.connect_nodes(8, 9)
-        self.connect_nodes(9, 8)
 
         self.sync_all()
 
@@ -79,10 +72,6 @@ def run_test(self):
         three_peers = self.nodes[3].getpeerinfo()
         four_peers = self.nodes[4].getpeerinfo()
         five_peers = self.nodes[5].getpeerinfo()
-        six_peers = self.nodes[6].getpeerinfo()
-        seven_peers = self.nodes[7].getpeerinfo()
-        eight_peers = self.nodes[8].getpeerinfo()
-        nine_peers = self.nodes[9].getpeerinfo()
 
         for tank in self.warnet.tanks:
             self.log.info(
@@ -104,13 +93,6 @@ def run_test(self):
         self.assert_connection(four_peers, 5, ConnectionType.IP)
         self.assert_connection(five_peers, 3, ConnectionType.IP)
         self.assert_connection(five_peers, 4, ConnectionType.DNS)
-        self.assert_connection(five_peers, 6, ConnectionType.DNS)
-        self.assert_connection(six_peers, 5, ConnectionType.IP)
-        self.assert_connection(six_peers, 7, ConnectionType.DNS)
-        self.assert_connection(seven_peers, 6, ConnectionType.IP)
-        # Check the pre-connected nodes
-        self.assert_connection(eight_peers, 9, ConnectionType.DNS)
-        self.assert_connection(nine_peers, 8, ConnectionType.IP)
 
         self.log.info(
             f"Successfully ran the connect_dag.py scenario using a temporary file: "