image build: use cmake with docker buildx bake

.github/workflows/test.yml

@@ -44,11 +44,15 @@ jobs:
           - dag_connection_test.py
           - graph_test.py
           - logging_test.py
+          - ln_basic_test.py
+          - ln_test.py
           - rpc_test.py
           - services_test.py
           - signet_test.py
+          - simln_test.py
           - scenarios_test.py
           - namespace_admin_test.py
+          - wargames_test.py
     steps:
       - uses: actions/checkout@v4
       - uses: azure/[email protected]

.gitignore

@@ -7,3 +7,4 @@ warnet.egg-info
 dist/
 build/
 **/kubeconfigs/
+src/warnet/_version.py

README.md

@@ -17,6 +17,7 @@ Monitor and analyze the emergent behaviors of Bitcoin networks.
 - [Installation](/docs/install.md)
 - [CLI Commands](/docs/warnet.md)
 - [Network configuration with yaml files](/docs/config.md)
+- [Plugins](/docs/plugins.md)
 - [Scenarios](/docs/scenarios.md)
 - [Monitoring](/docs/logging_monitoring.md)
 - [Snapshots](/docs/snapshots.md)

docker-bake.hcl

@@ -0,0 +1,238 @@
+group "all" {
+  targets = [
+    "bitcoin-28",
+    "bitcoin-27",
+    "bitcoin-26",
+    "v0-21-1",
+    "v0-20-0",
+    "v0-19-2",
+    "v0-17-0",
+    "v0-16-1",
+    "bitcoin-unknown-message",
+    "bitcoin-invalid-blocks",
+    "bitcoin-50-orphans",
+    "bitcoin-no-mp-trim",
+    "bitcoin-disabled-opcodes",
+    "bitcoin-5k-inv"
+  ]
+}
+
+group "maintained" {
+  targets = [
+    "bitcoin-28",
+    "bitcoin-27",
+    "bitcoin-26"
+  ]
+}
+
+group "practice" {
+  targets = [
+    "bitcoin-unknown-message",
+    "bitcoin-invalid-blocks",
+    "bitcoin-50-orphans",
+    "bitcoin-no-mp-trim",
+    "bitcoin-disabled-opcodes",
+    "bitcoin-5k-inv"
+  ]
+}
+
+group "vulnerable" {
+  targets = [
+    "v0-21-1",
+    "v0-20-0",
+    "v0-19-2",
+    "v0-17-0",
+    "v0-16-1",
+  ]
+}
+
+target "maintained-base" {
+  context = "./resources/images/bitcoin"
+  args = {
+    REPO = "bitcoin/bitcoin"
+    BUILD_ARGS = "--disable-tests --without-gui --disable-bench --disable-fuzz-binary --enable-suppress-external-warnings"
+  }
+  platforms = ["linux/amd64", "linux/arm64", "linux/arm/v7"]
+}
+
+target "cmake-base" {
+  inherits = ["maintained-base"]
+  dockerfile = "./Dockerfile.dev"
+  args = {
+    BUILD_ARGS = "-DBUILD_TESTS=OFF -DBUILD_GUI=OFF -DBUILD_BENCH=OFF -DBUILD_FUZZ_BINARY=OFF -DWITH_ZMQ=ON"
+  }
+}
+
+target "autogen-base" {
+  inherits = ["maintained-base"]
+  dockerfile = "./Dockerfile"
+}
+
+target "bitcoin-master" {
+  inherits = ["cmake-base"]
+  tags = ["bitcoindevproject/bitcoin:28.1"]
+  args = {
+    COMMIT_SHA = "bd0ee07310c3dcdd08633c69eac330e2e567b235"
+  }
+}
+
+target "bitcoin-28" {
+  inherits = ["autogen-base"]
+  tags = ["bitcoindevproject/bitcoin:28.0"]
+  args = {
+    COMMIT_SHA = "110183746150428e6385880c79f8c5733b1361ba"
+  }
+}
+
+target "bitcoin-27" {
+  inherits = ["autogen-base"]
+  tags = ["bitcoindevproject/bitcoin:27.2"]
+  args = {
+    COMMIT_SHA = "bf03c458e994abab9be85486ed8a6d8813313579"
+  }
+}
+
+target "bitcoin-26" {
+  inherits = ["autogen-base"]
+  tags = ["bitcoindevproject/bitcoin:26.2"]
+  args = {
+    COMMIT_SHA = "7b7041019ba5e7df7bde1416aa6916414a04f3db"
+  }
+}
+
+target "practice-base" {
+  dockerfile = "./Dockerfile"
+  context = "./resources/images/bitcoin/insecure"
+  contexts = {
+      bitcoin-src = "."
+  }
+  args = {
+    ALPINE_VERSION = "3.20"
+    BITCOIN_VERSION = "28.1.1"
+    EXTRA_PACKAGES = "sqlite-dev"
+    EXTRA_RUNTIME_PACKAGES = ""
+    REPO = "willcl-ark/bitcoin"
+  }
+  platforms = ["linux/amd64", "linux/armhf"]
+}
+
+target "bitcoin-unknown-message" {
+  inherits = ["practice-base"]
+  tags = ["bitcoindevproject/bitcoin:99.0.0-unknown-message"]
+  args = {
+    COMMIT_SHA = "ae999611026e941eca5c0b61f22012c3b3f3d8dc"
+  }
+}
+
+target "bitcoin-invalid-blocks" {
+  inherits = ["practice-base"]
+  tags = ["bitcoindevproject/bitcoin:98.0.0-invalid-blocks"]
+  args = {
+    COMMIT_SHA = "9713324368e5a966ec330389a533ae8ad7a0ea8f"
+  }
+}
+
+target "bitcoin-50-orphans" {
+  inherits = ["practice-base"]
+  tags = ["bitcoindevproject/bitcoin:97.0.0-50-orphans"]
+  args = {
+    COMMIT_SHA = "cbcb308eb29621c0db3a105e1a1c1788fb0dab6b"
+  }
+}
+
+target "bitcoin-no-mp-trim" {
+  inherits = ["practice-base"]
+  tags = ["bitcoindevproject/bitcoin:96.0.0-no-mp-trim"]
+  args = {
+    COMMIT_SHA = "a3a15a9a06dd541d1dafba068c00eedf07e1d5f8"
+  }
+}
+
+target "bitcoin-disabled-opcodes" {
+  inherits = ["practice-base"]
+  tags = ["bitcoindevproject/bitcoin:95.0.0-disabled-opcodes"]
+  args = {
+    COMMIT_SHA = "5bdb8c52a8612cac9aa928c84a499dd701542b2a"
+  }
+}
+
+target "bitcoin-5k-inv" {
+  inherits = ["practice-base"]
+  tags = ["bitcoindevproject/bitcoin:94.0.0-5k-inv"]
+  args = {
+    COMMIT_SHA = "e70e610e07eea3aeb0c49ae0bd9f4049ffc1b88c"
+  }
+}
+
+target "CVE-base" {
+  dockerfile = "./Dockerfile"
+  context = "./resources/images/bitcoin/insecure"
+  contexts = {
+      bitcoin-src = "."
+  }
+  platforms = ["linux/amd64", "linux/armhf"]
+  args = {
+    REPO = "josibake/bitcoin"
+  }
+}
+
+target "v0-16-1" {
+  inherits = ["CVE-base"]
+  tags = ["bitcoindevproject/bitcoin:0.16.1"]
+  args = {
+    ALPINE_VERSION = "3.7"
+    BITCOIN_VERSION = "0.16.1"
+    COMMIT_SHA = "dc94c00e58c60412a4e1a540abdf0b56093179e8"
+    EXTRA_PACKAGES = "protobuf-dev libressl-dev"
+    EXTRA_RUNTIME_PACKAGES = "boost boost-program_options libressl"
+    PRE_CONFIGURE_COMMANDS = "sed -i '/AC_PREREQ/a\\AR_FLAGS=cr' src/univalue/configure.ac && sed -i '/AX_PROG_CC_FOR_BUILD/a\\AR_FLAGS=cr' src/secp256k1/configure.ac && sed -i 's:sys/fcntl.h:fcntl.h:' src/compat.h"
+  }
+}
+
+target "v0-17-0" {
+  inherits = ["CVE-base"]
+  tags = ["bitcoindevproject/bitcoin:0.17.0"]
+  args = {
+    ALPINE_VERSION = "3.9"
+    BITCOIN_VERSION = "0.17.0"
+    COMMIT_SHA = "f6b2db49a707e7ad433d958aee25ce561c66521a"
+    EXTRA_PACKAGES = "protobuf-dev libressl-dev"
+    EXTRA_RUNTIME_PACKAGES = "boost boost-program_options libressl sqlite-dev"
+  }
+}
+
+target "v0-19-2" {
+  inherits = ["CVE-base"]
+  tags = ["bitcoindevproject/bitcoin:0.19.2"]
+  args = {
+    ALPINE_VERSION = "3.12.12"
+    BITCOIN_VERSION = "0.19.2"
+    COMMIT_SHA = "e20f83eb5466a7d68227af14a9d0cf66fb520ffc"
+    EXTRA_PACKAGES = "sqlite-dev libressl-dev"
+    EXTRA_RUNTIME_PACKAGES = "boost boost-program_options libressl sqlite-dev"
+  }
+}
+
+target "v0-20-0" {
+  inherits = ["CVE-base"]
+  tags = ["bitcoindevproject/bitcoin:0.20.0"]
+  args = {
+    ALPINE_VERSION = "3.12.12"
+    BITCOIN_VERSION = "0.20.0"
+    COMMIT_SHA = "0bbff8feff0acf1693dfe41184d9a4fd52001d3f"
+    EXTRA_PACKAGES = "sqlite-dev miniupnpc-dev"
+    EXTRA_RUNTIME_PACKAGES = "boost-filesystem miniupnpc-dev sqlite-dev"
+  }
+}
+
+target "v0-21-1" {
+  inherits = ["CVE-base"]
+  tags = ["bitcoindevproject/bitcoin:0.21.1"]
+  args = {
+    ALPINE_VERSION = "3.17"
+    BITCOIN_VERSION = "0.21.1"
+    COMMIT_SHA = "e0a22f14c15b4877ef6221f9ee2dfe510092d734"
+    EXTRA_PACKAGES = "sqlite-dev"
+    EXTRA_RUNTIME_PACKAGES = "boost-filesystem sqlite-dev"
+  }
+}

docs/developer-notes.md

@@ -72,3 +72,20 @@ python3 -m build
 # Upload to Pypi
 python3 -m twine upload dist/*
 ```
+
+## Building docker images
+
+The Bitcoin Core docker images used by warnet are specified in the *docker-bake.hcl* file.
+This uses the (experimental) `bake` build functionality of docker buildx.
+We use [HCL language](https://github.com/hashicorp/hcl) in the declaration file itself.
+See the `bake` [documentation](https://docs.docker.com/build/bake/) for more information on specifications, and how to e.g. override arguments.
+
+In order to build (or "bake") a certain image, find the image's target (name) in the *docker-bake.hcl* file, and then run `docker buildx bake <target>`.
+
+```bash
+# build the  dummy image that will crash on 5k invs
+docker buildx bake bitcoin-5k-inv
+
+# build the same image, but set platform to only linux/amd64
+docker buildx bake bitcoin-5k-inv --set bitcoin-5k-inv.platform=linux/amd64
+```

docs/logging_monitoring.md

@@ -96,7 +96,7 @@ For example, the default metrics listed above would be explicitly configured as
 nodes:
   - name: tank-0000
     metricsExport: true
-    metrics: blocks=getblockcount() inbounds=getnetworkinfo()["connections_in"] outbounds=getnetworkinfo()["connections_in"] mempool_size=getmempoolinfo()["size"]
+    metrics: blocks=getblockcount() inbounds=getnetworkinfo()["connections_in"] outbounds=getnetworkinfo()["connections_out"] mempool_size=getmempoolinfo()["size"]
 ```
 
 The data can be retrieved directly from the Prometheus exporter container in the tank pod via port `9332`, example:
@@ -108,7 +108,7 @@ blocks 704.0
 # HELP inbounds getnetworkinfo()["connections_in"]
 # TYPE inbounds gauge
 inbounds 0.0
-# HELP outbounds getnetworkinfo()["connections_in"]
+# HELP outbounds getnetworkinfo()["connections_out"]
 # TYPE outbounds gauge
 outbounds 0.0
 # HELP mempool_size getmempoolinfo()["size"]

docs/plugins.md

@@ -0,0 +1,72 @@
+# Plugins
+
+Plugins extend Warnet. Plugin authors can import commands from Warnet and interact with the kubernetes cluster, and plugin users can run plugins from the command line or from the `network.yaml` file.
+
+## Activating plugins from `network.yaml`
+
+You can activate a plugin command by placing it in the `plugins` section at the bottom of each `network.yaml` file like so:
+
+````yaml
+nodes:
+  <<snip>>
+
+plugins:  # This marks the beginning of the plugin section
+  preDeploy:  # This is a hook. This particular hook will call plugins before deploying anything else.
+    hello:  # This is the name of the plugin.
+      entrypoint: "../plugins/hello"  # Every plugin must specify a path to its entrypoint.
+      podName: "hello-pre-deploy"  # Plugins can have their own particular configurations, such as how to name a pod.
+      helloTo: "preDeploy!"  # This configuration tells the hello plugin who to say "hello" to.
+````
+
+## Many kinds of hooks
+There are many hooks to the Warnet `deploy` command. The example below specifies them:
+
+````yaml
+nodes:
+  <<snip>>
+
+plugins:
+  preDeploy:  # Plugins will run before any other `deploy` code.
+    hello:
+      entrypoint: "../plugins/hello"
+      podName: "hello-pre-deploy"
+      helloTo: "preDeploy!"
+  postDeploy:  # Plugins will run after all the `deploy` code has run.
+    simln:
+      entrypoint: "../plugins/simln"
+      activity: '[{"source": "tank-0003-ln", "destination": "tank-0005-ln", "interval_secs": 1, "amount_msat": 2000}]'
+    hello:
+      entrypoint: "../plugins/hello"
+      podName: "hello-post-deploy"
+      helloTo: "postDeploy!"
+  preNode:  # Plugins will run before `deploy` launches a node (once per node).
+    hello:
+      entrypoint: "../plugins/hello"
+      helloTo: "preNode!"
+  postNode:  # Plugins will run after `deploy` launches a node (once per node).
+    hello:
+      entrypoint: "../plugins/hello"
+      helloTo: "postNode!"
+  preNetwork:  # Plugins will run before `deploy` launches the network (essentially between logging and when nodes are deployed)
+    hello:
+      entrypoint: "../plugins/hello"
+      helloTo: "preNetwork!"
+      podName: "hello-pre-network"
+  postNetwork:  # Plugins will run after the network deploy threads have been joined.
+    hello:
+      entrypoint: "../plugins/hello"
+      helloTo: "postNetwork!"
+      podName: "hello-post-network"
+````
+
+Warnet will execute these plugin commands during each invocation of `warnet deploy`.
+
+
+
+## A "hello" example
+
+To get started with an example plugin, review the `README` of the `hello` plugin found in any initialized Warnet directory:
+
+1. `warnet init`
+2. `cd plugins/hello/`
+