diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0524bed9c..7c7a71a2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,6 +59,8 @@ jobs: - uses: azure/setup-helm@v4.2.0 - uses: medyagh/setup-minikube@master with: + cpus: max + memory: 10000m mount-path: ${{ github.workspace }}:/mnt/src - uses: actions/download-artifact@v4 with: diff --git a/src/warnet/backend/kubernetes_backend.py b/src/warnet/backend/kubernetes_backend.py index f8821888d..a796096ab 100644 --- a/src/warnet/backend/kubernetes_backend.py +++ b/src/warnet/backend/kubernetes_backend.py @@ -384,33 +384,62 @@ def create_bitcoind_container(self, tank: Tank) -> client.V1Container: bitcoind_options = tank.get_bitcoin_conf(peers) container_env = [client.V1EnvVar(name="BITCOIN_ARGS", value=bitcoind_options)] - bitcoind_container = client.V1Container( - name=container_name, - image=container_image, - env=container_env, - liveness_probe=client.V1Probe( - failure_threshold=3, - initial_delay_seconds=5, - period_seconds=5, - timeout_seconds=1, - _exec=client.V1ExecAction(command=["pidof", "bitcoind"]), - ), - readiness_probe=client.V1Probe( - failure_threshold=1, - initial_delay_seconds=0, - period_seconds=1, - timeout_seconds=1, - tcp_socket=client.V1TCPSocketAction(port=tank.rpc_port), - ), - security_context=client.V1SecurityContext( - privileged=True, - capabilities=client.V1Capabilities(add=["NET_ADMIN", "NET_RAW"]), - ), + if tank.resources: + resources = client.V1ResourceRequirements( + requests=tank.resources["requests"], + limits=tank.resources["limits"], + ) + else: + resources = client.V1ResourceRequirements() + + liveness_probe = client.V1Probe( + failure_threshold=3, + initial_delay_seconds=5, + period_seconds=5, + timeout_seconds=1, + _exec=client.V1ExecAction(command=["pidof", "bitcoind"]), ) - self.log.debug( - f"Created bitcoind container for tank {tank.index} using {bitcoind_options=:}" + readiness_probe = client.V1Probe( + failure_threshold=1, + initial_delay_seconds=0, + period_seconds=1, + timeout_seconds=1, + tcp_socket=client.V1TCPSocketAction(port=tank.rpc_port), ) - return bitcoind_container + security_context = client.V1SecurityContext( + privileged=True, + capabilities=client.V1Capabilities(add=["NET_ADMIN", "NET_RAW"]), + ) + if tank.resources: + bitcoind_container = client.V1Container( + name=container_name, + image=container_image, + env=container_env, + liveness_probe=liveness_probe, + readiness_probe=readiness_probe, + security_context=security_context, + resources=client.V1ResourceRequirements( + requests=tank.resources["requests"], + limits=tank.resources["limits"], + ), + ) + self.log.debug( + f"Created bitcoind container for tank {tank.index} using {bitcoind_options=:}" + ) + return bitcoind_container + else: + bitcoind_container = client.V1Container( + name=container_name, + image=container_image, + env=container_env, + liveness_probe=liveness_probe, + readiness_probe=readiness_probe, + security_context=security_context, + ) + self.log.debug( + f"Created bitcoind container for tank {tank.index} using {bitcoind_options=:}" + ) + return bitcoind_container def create_prometheus_container(self, tank) -> client.V1Container: env = [ diff --git a/src/warnet/cli/graph.py b/src/warnet/cli/graph.py index 45128d603..628772ee5 100644 --- a/src/warnet/cli/graph.py +++ b/src/warnet/cli/graph.py @@ -40,7 +40,8 @@ def create(number: int, outfile: Path, version: str, bitcoin_conf: Path, random: @click.option("--outfile", type=click.Path()) @click.option("--cb", type=str) @click.option("--ln_image", type=str) -def import_json(infile: Path, outfile: Path, cb: str, ln_image: str): +@click.option("--ci", is_flag=True) +def import_json(infile: Path, outfile: Path, cb: str, ln_image: str, ci: bool = False): """ Create a cycle graph with nodes imported from lnd `describegraph` JSON file, and additionally include 7 extra random outbounds per node. Include lightning @@ -64,6 +65,8 @@ def import_json(infile: Path, outfile: Path, cb: str, ln_image: str): graph.nodes[n]["ln_cb_image"] = cb if ln_image: graph.nodes[n]["ln_image"] = ln_image + if ci: + graph.nodes[n]["resources"] = "{}" # Save a map of LN pubkey -> Tank index ln_ids = {} diff --git a/src/warnet/graph_schema.json b/src/warnet/graph_schema.json index ac1f7aa9f..27766c432 100644 --- a/src/warnet/graph_schema.json +++ b/src/warnet/graph_schema.json @@ -16,52 +16,62 @@ "node": { "type": "object", "properties": { - "version": { - "type": "string", - "comment": "Bitcoin Core version with an available Warnet tank image on Dockerhub. May also be a GitHub repository with format user/repository:branch to build from source code"}, "image": { "type": "string", - "comment": "Bitcoin Core Warnet tank image on Dockerhub with the format repository/image:tag"}, + "default": "bitcoindevproject/bitcoin:27.0", + "comment": "Bitcoin Core Warnet tank image on Dockerhub with the format repository/image:tag" + }, "bitcoin_config": { "type": "string", "default": "", - "comment": "A string of Bitcoin Core options in command-line format, e.g. '-debug=net -blocksonly'"}, + "comment": "A string of Bitcoin Core options in command-line format, e.g. '-debug=net -blocksonly'" + }, "tc_netem": { "type": "string", - "comment": "A tc-netem command as a string beginning with 'tc qdisc add dev eth0 root netem'"}, + "comment": "A tc-netem command as a string beginning with 'tc qdisc add dev eth0 root netem'" + }, "exporter": { "type": "boolean", "default": false, - "comment": "Whether to attach a Prometheus data exporter to the tank"}, + "comment": "Whether to attach a Prometheus data exporter to the tank" + }, "metrics": { "type": "string", - "comment": "A space-separated string of RPC queries to scrape by Prometheus"}, + "comment": "A space-separated string of RPC queries to scrape by Prometheus" + }, "collect_logs": { "type": "boolean", - "default": false, - "comment": "Whether to collect Bitcoin Core debug logs with Promtail"}, + "default": true, + "comment": "Whether to collect Bitcoin Core debug logs with Promtail" + }, "build_args": { "type": "string", "default": "", - "comment": "A string of configure options used when building Bitcoin Core from source code, e.g. '--without-gui --disable-tests'"}, + "comment": "A string of configure options used when building Bitcoin Core from source code, e.g. '--without-gui --disable-tests'" + }, "ln": { "type": "string", - "comment": "Attach a lightning network node of this implementation (currently only supports 'lnd' or 'cln')"}, + "comment": "Attach a lightning network node of this implementation (currently only supports 'lnd' or 'cln')" + }, "ln_image": { "type": "string", - "comment": "Specify a lightning network node image from Dockerhub with the format repository/image:tag"}, + "comment": "Specify a lightning network node image from Dockerhub with the format repository/image:tag" + }, "ln_cb_image": { "type": "string", - "comment": "Specify a lnd Circuit Breaker image from Dockerhub with the format repository/image:tag"}, + "comment": "Specify a lnd Circuit Breaker image from Dockerhub with the format repository/image:tag" + }, "ln_config": { "type": "string", - "comment": "A string of arguments for the lightning network node in command-line format, e.g. '--protocol.wumbo-channels --bitcoin.timelockdelta=80'"} + "comment": "A string of arguments for the lightning network node in command-line format, e.g. '--protocol.wumbo-channels --bitcoin.timelockdelta=80'" + }, + "resources": { + "type": "string", + "comment": "Kubernetes resource requests and limits for the node", + "default": "{'requests':{'cpu':'500m', 'memory':'500Mi'}, 'limits':{'cpu':'1000m', 'memory':'1500Mi'}}" + } }, - "additionalProperties": false, - "oneOf": [ - {"required": ["version"]}, - {"required": ["image"]} - ], + "additionalProperties": false, "required": [] }, "edge": { @@ -69,13 +79,16 @@ "properties": { "channel_open": { "type": "string", - "comment": "Indicate that this edge is a lightning channel with these arguments passed to lnd openchannel"}, + "comment": "Indicate that this edge is a lightning channel with these arguments passed to lnd openchannel" + }, "source_policy": { "type": "string", - "comment": "Update the channel originator policy by passing these arguments passed to lnd updatechanpolicy"}, + "comment": "Update the channel originator policy by passing these arguments passed to lnd updatechanpolicy" + }, "target_policy": { "type": "string", - "comment": "Update the channel partner policy by passing these arguments passed to lnd updatechanpolicy"} + "comment": "Update the channel partner policy by passing these arguments passed to lnd updatechanpolicy" + } }, "additionalProperties": false, "required": [] diff --git a/src/warnet/tank.py b/src/warnet/tank.py index ac04d3f70..a9c9303de 100644 --- a/src/warnet/tank.py +++ b/src/warnet/tank.py @@ -46,6 +46,7 @@ def __init__(self, index: int, warnet): self.bitcoin_network = warnet.bitcoin_network self.version: str = "" self.image: str = "" + self.resources: dict = dict() self.bitcoin_config = "" self.netem = None self.exporter = False @@ -81,7 +82,10 @@ def parse_graph_node(self, node): value = node.get(property, specs.get("default")) if property == "version": self._parse_version(value) - setattr(self, property, value) + if property == "resources": + setattr(self, property, eval(value)) + else: + setattr(self, property, value) graph_properties[property] = value if self.version and self.image: diff --git a/src/warnet/utils.py b/src/warnet/utils.py index 23dad566b..eb13e57f0 100644 --- a/src/warnet/utils.py +++ b/src/warnet/utils.py @@ -421,15 +421,8 @@ def create_cycle_graph(n: int, version: str, bitcoin_conf: str | None, random_ve conf_contents = dump_bitcoin_conf(conf_dict, for_graph=True) # populate our custom fields - for i, node in enumerate(graph.nodes()): - if random_version: - graph.nodes[node]["version"] = random.choice(WEIGHTED_TAGS) - else: - # One node demoing the image tag - if i == 1: - graph.nodes[node]["image"] = f"bitcoindevproject/bitcoin:{version}" - else: - graph.nodes[node]["version"] = version + for node in graph.nodes(): + graph.nodes[node]["image"] = f"bitcoindevproject/bitcoin:{version}" graph.nodes[node]["bitcoin_config"] = conf_contents graph.nodes[node]["tc_netem"] = "" graph.nodes[node]["build_args"] = "" diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index 32c2ccc8c..675171381 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -10,7 +10,7 @@ class DAGConnectionTest(TestBase): def __init__(self): super().__init__() self.graph_file_path = ( - Path(os.path.dirname(__file__)) / "data" / "ten_semi_unconnected.graphml" + Path(os.path.dirname(__file__)) / "data" / "six_semi_unconnected.graphml" ) def run_test(self): diff --git a/test/data/12_node_ring.graphml b/test/data/6_node_ring.graphml similarity index 62% rename from test/data/12_node_ring.graphml rename to test/data/6_node_ring.graphml index 7cdf3a7f7..6976bc904 100644 --- a/test/data/12_node_ring.graphml +++ b/test/data/6_node_ring.graphml @@ -1,6 +1,5 @@ - @@ -17,51 +16,27 @@ - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w0 -debug=validation - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w1 -debug=validation - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w2 -debug=validation - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w3 - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w4 - 27.0 - -uacomment=w5 - - - 27.0 - -uacomment=w6 - - - 27.0 - -uacomment=w7 - - - 27.0 - -uacomment=w8 - - - 27.0 - -uacomment=w9 - - - 27.0 - -uacomment=w10 - - - 27.0 + bitcoindevproject/bitcoin:27.0 @@ -70,12 +45,6 @@ - - - - - - - + diff --git a/test/data/ln.graphml b/test/data/ln.graphml index e0606c93f..4a4d9b9ec 100644 --- a/test/data/ln.graphml +++ b/test/data/ln.graphml @@ -1,6 +1,5 @@ - @@ -18,34 +17,34 @@ simln - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w0 lnd lightninglabs/lnd:v0.17.5-beta true - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w1 lnd pinheadmz/circuitbreaker:278737d true - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w2 lnd pinheadmz/circuitbreaker:278737d --bitcoin.timelockdelta=33 - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w2 cln --cltv-delta=33 - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w3 @@ -68,4 +67,4 @@ feebase=5500 feeppm=3 - \ No newline at end of file + diff --git a/test/data/logging.graphml b/test/data/logging.graphml index 54b3b73cb..bba85493b 100644 --- a/test/data/logging.graphml +++ b/test/data/logging.graphml @@ -1,6 +1,5 @@ - @@ -17,16 +16,16 @@ - 27.0 + bitcoindevproject/bitcoin:27.0 true - 27.0 + bitcoindevproject/bitcoin:27.0 true txrate=getchaintxstats(10)["txrate"] - 27.0 + bitcoindevproject/bitcoin:27.0 diff --git a/test/data/permutations.graphml b/test/data/permutations.graphml index 0c4686f61..83e48567a 100644 --- a/test/data/permutations.graphml +++ b/test/data/permutations.graphml @@ -1,6 +1,5 @@ - @@ -17,7 +16,7 @@ - 27.0 + bitcoindevproject/bitcoin:27.0 @@ -33,7 +32,7 @@ False - 27.0 + bitcoindevproject/bitcoin:27.0 @@ -41,7 +40,7 @@ False - 27.0 + bitcoindevproject/bitcoin:27.0 @@ -49,7 +48,7 @@ False - 27.0 + bitcoindevproject/bitcoin:27.0 @@ -57,23 +56,7 @@ False - 27.0 - - - - False - False - - - 27.0 - - - - False - False - - - 27.0 + bitcoindevproject/bitcoin:27.0 @@ -87,7 +70,5 @@ - - - \ No newline at end of file + diff --git a/test/data/scenario_connect_dag.py b/test/data/scenario_connect_dag.py index f0565f9b7..0768f7d44 100644 --- a/test/data/scenario_connect_dag.py +++ b/test/data/scenario_connect_dag.py @@ -20,7 +20,7 @@ class ConnectionType(Enum): class ConnectDag(WarnetTestFramework): def set_test_params(self): # This is just a minimum - self.num_nodes = 10 + self.num_nodes = 6 def add_options(self, parser): parser.add_argument( @@ -63,13 +63,6 @@ def run_test(self): self.connect_nodes(2, 4) self.connect_nodes(3, 5) self.connect_nodes(5, 4) - self.connect_nodes(5, 6) - self.connect_nodes(6, 7) - - # Nodes 8 & 9 shall come pre-connected. Attempt to connect them anyway to test the handling - # of dns node addresses - self.connect_nodes(8, 9) - self.connect_nodes(9, 8) self.sync_all() @@ -79,10 +72,6 @@ def run_test(self): three_peers = self.nodes[3].getpeerinfo() four_peers = self.nodes[4].getpeerinfo() five_peers = self.nodes[5].getpeerinfo() - six_peers = self.nodes[6].getpeerinfo() - seven_peers = self.nodes[7].getpeerinfo() - eight_peers = self.nodes[8].getpeerinfo() - nine_peers = self.nodes[9].getpeerinfo() for tank in self.warnet.tanks: self.log.info( @@ -104,13 +93,6 @@ def run_test(self): self.assert_connection(four_peers, 5, ConnectionType.IP) self.assert_connection(five_peers, 3, ConnectionType.IP) self.assert_connection(five_peers, 4, ConnectionType.DNS) - self.assert_connection(five_peers, 6, ConnectionType.DNS) - self.assert_connection(six_peers, 5, ConnectionType.IP) - self.assert_connection(six_peers, 7, ConnectionType.DNS) - self.assert_connection(seven_peers, 6, ConnectionType.IP) - # Check the pre-connected nodes - self.assert_connection(eight_peers, 9, ConnectionType.DNS) - self.assert_connection(nine_peers, 8, ConnectionType.IP) self.log.info( f"Successfully ran the connect_dag.py scenario using a temporary file: " diff --git a/test/data/services.graphml b/test/data/services.graphml index c9e0a0d01..e9bf55ca0 100644 --- a/test/data/services.graphml +++ b/test/data/services.graphml @@ -1,6 +1,5 @@ - @@ -12,16 +11,18 @@ + - 27.0 + bitcoindevproject/bitcoin:27.0 -uacomment=w0 -debug=validation true lnd + {'requests':{'cpu':'25m', 'memory':'100Mi'}, 'limits':{'cpu':'100m', 'memory':'250Mi'}} diff --git a/test/data/ten_semi_unconnected.graphml b/test/data/six_semi_unconnected.graphml similarity index 68% rename from test/data/ten_semi_unconnected.graphml rename to test/data/six_semi_unconnected.graphml index c2277407c..2f978df6e 100644 --- a/test/data/ten_semi_unconnected.graphml +++ b/test/data/six_semi_unconnected.graphml @@ -1,6 +1,5 @@ - @@ -17,7 +16,7 @@ - 26.0 + bitcoindevproject/bitcoin:26.0 @@ -33,7 +32,7 @@ False - 26.0 + bitcoindevproject/bitcoin:26.0 @@ -41,7 +40,7 @@ False - 26.0 + bitcoindevproject/bitcoin:26.0 @@ -49,7 +48,7 @@ False - 26.0 + bitcoindevproject/bitcoin:26.0 @@ -57,45 +56,13 @@ False - 26.0 - - - - False - False - - - 26.0 - - - - False - False - - - 26.0 - - - - False - False - - - 26.0 - - - - False - False - - - 26.0 + bitcoindevproject/bitcoin:26.0 False False - + diff --git a/test/graph_test.py b/test/graph_test.py index 68485f93d..15da68fe4 100755 --- a/test/graph_test.py +++ b/test/graph_test.py @@ -35,17 +35,13 @@ def run_test(self): def test_graph_creation_and_import(self): self.log.info(f"CLI tool creating test graph file: {self.tf_create}") - self.log.info( - self.warcli( - f"graph create 10 --outfile={self.tf_create} --version={DEFAULT_TAG}", network=False - ) - ) + self.log.info(self.warcli(f"graph create 10 --outfile={self.tf_create}", network=False)) self.wait_for_predicate(lambda: Path(self.tf_create).exists()) self.log.info(f"CLI tool importing json and writing test graph file: {self.tf_import}") self.log.info( self.warcli( - f"graph import-json {self.json_file_path} --outfile={self.tf_import} --ln_image=carlakirkcohen/lnd:attackathon --cb=carlakirkcohen/circuitbreaker:attackathon-test", + f"graph import-json {self.json_file_path} --outfile={self.tf_import} --ln_image=carlakirkcohen/lnd:attackathon --cb=carlakirkcohen/circuitbreaker:attackathon-test --ci", network=False, ) ) diff --git a/test/onion_test.py b/test/onion_test.py index 7f7454b60..7b5d7fd18 100755 --- a/test/onion_test.py +++ b/test/onion_test.py @@ -10,7 +10,7 @@ class OnionTest(TestBase): def __init__(self): super().__init__() - self.graph_file_path = Path(os.path.dirname(__file__)) / "data" / "12_node_ring.graphml" + self.graph_file_path = Path(os.path.dirname(__file__)) / "data" / "6_node_ring.graphml" self.onion_addr = None def run_test(self): diff --git a/test/rpc_test.py b/test/rpc_test.py index 642151e5f..fa2c5dd94 100755 --- a/test/rpc_test.py +++ b/test/rpc_test.py @@ -10,7 +10,7 @@ class RPCTest(TestBase): def __init__(self): super().__init__() - self.graph_file_path = Path(os.path.dirname(__file__)) / "data" / "12_node_ring.graphml" + self.graph_file_path = Path(os.path.dirname(__file__)) / "data" / "6_node_ring.graphml" def run_test(self): self.start_server() diff --git a/test/scenarios_test.py b/test/scenarios_test.py index 9e87f0fb3..da8a0a2f4 100755 --- a/test/scenarios_test.py +++ b/test/scenarios_test.py @@ -9,7 +9,7 @@ class ScenariosTest(TestBase): def __init__(self): super().__init__() - self.graph_file_path = Path(os.path.dirname(__file__)) / "data" / "12_node_ring.graphml" + self.graph_file_path = Path(os.path.dirname(__file__)) / "data" / "6_node_ring.graphml" def run_test(self): try: