diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 348551aac..d84b9f6b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,12 +16,9 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - # - uses: chartboost/ruff-action@v1 - # Until this gets updated we need to use this commit hash (or later) - - uses: chartboost/ruff-action@491342200cdd1cf4d5132a30ddc546b3b5bc531b + - uses: chartboost/ruff-action@v1 with: args: 'format --check' - changed-files: 'true' build-image: needs: [ruff, ruff-format] runs-on: ubuntu-latest @@ -52,11 +49,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - test: [scenarios_test.py, rpc_test.py, graph_test.py, ln_test.py, dag_connection_test.py] + test: [scenarios_test.py, rpc_test.py, graph_test.py, ln_test.py, dag_connection_test.py, logging_test.py] steps: - uses: actions/checkout@v4 - uses: hynek/setup-cached-uv@v1 - uses: medyagh/setup-minikube@master + - uses: azure/setup-helm@v4.2.0 with: mount-path: ${{ github.workspace }}:/mnt/src - uses: actions/download-artifact@v4 diff --git a/docs/monitoring.md b/docs/monitoring.md index 122483c3f..31678bebf 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -1,32 +1,83 @@ # Monitoring -## Monitoring container resource usage +## Prometheus -When run in docker, a few additional containers are started up: +To monitor RPC return values over time, a Prometheus data exporter can be connected +to any Bitcoin Tank and configured to scrape any available RPC results. -* CAdvisor (container Monitoring) -* Prometheus (log scraper) -* Grafana (graphing/dashboard tool) +The `bitcoin-exporter` image is defined in `resources/images/exporter` and +maintained in the BitcoinDevProject dockerhub organization. To add the exporter +in the Tank pod with Bitcoin Core add the `"exporter"` key to the node in the graphml file: -## CAdvisor +```xml + + 27.0 + true + +``` -CAdvisor needs no additional setup, and can be accessed from the docker host at -localhost:8080 +The default metrics are defined in the `bitcoin-exporter` image: +- Block count +- Number of inbound peers +- Number of outbound peers +- Mempool size (# of TXs) -## Prometheus +Metrics can be configured by setting a `"metrics"` key to the node in the graphml file. +The metrics value is a space-separated list of labels, RPC commands with arguments, and +JSON keys to resolve the desired data: + +``` +label=method(arguments)[JSON result key][...] +``` + +For example, the default metrics listed above are defined as: + +```xml + + 27.0 + true + blocks=getblockcount() inbounds=getnetworkinfo()["connections_in"] outbounds=getnetworkinfo()["connections_in"] mempool_size=getmempoolinfo()["size"] + +``` + +The data can be retrieved from the Prometheus exporter on port `9332`, example: -Prometheus should also not need any additional setup, and can be accessed from -the docker host at localhost:9090 +``` +# HELP blocks getblockcount() +# TYPE blocks gauge +blocks 704.0 +# HELP inbounds getnetworkinfo()["connections_in"] +# TYPE inbounds gauge +inbounds 0.0 +# HELP outbounds getnetworkinfo()["connections_in"] +# TYPE outbounds gauge +outbounds 0.0 +# HELP mempool_size getmempoolinfo()["size"] +# TYPE mempool_size gauge +mempool_size 0.0 +``` ## Grafana -Grafana is provisioned with a single default dashboard, but alternative -dashboards can be added or created. +Data from Prometheus exporters can be collected and fed into Grafana for a +web-based interface. + +### Install logging infrastructure + +First make sure you have `helm` installed, then run the `install_logging` script: + +```bash +resources/scripts/install_logging.sh +``` + +To forward port `3000` and view the Grafana dashboard run the `connect_logging` script: + +```bash +resources/scripts/connect_logging.sh +``` + +The Grafana dashboard (and API) will be accessible without requiring authentication +at http://localhost:3000 -Grafana can be accessed on the docker host from localhost:3000 using username -`admin` and password `admin` by default. -The default dashboard is called "Docker Container & Host Metrics" and can be -accessed via the "dashboards" tab, or from the bottom right of the home screen. -Additional dashboards and datasources may be added in the future. diff --git a/docs/running.md b/docs/running.md index aea8e0641..e8499953a 100644 --- a/docs/running.md +++ b/docs/running.md @@ -1,8 +1,7 @@ # Running Warnet -Warnet runs a server which can be used to manage multiple networks. On docker -this runs locally, but on Kubernetes this runs as a `statefulSet` in the -cluster. +Warnet runs a server which can be used to manage multiple networks. +In Kubernetes this runs as a `statefulSet` in the cluster. If the `$XDG_STATE_HOME` environment variable is set, the server will log to a file `$XDG_STATE_HOME/warnet/warnet.log`, otherwise it will use `$HOME/.warnet/warnet.log`. @@ -11,31 +10,9 @@ a file `$XDG_STATE_HOME/warnet/warnet.log`, otherwise it will use `$HOME/.warnet // TODO -### Install logging infrastructure +## Running large networks -First make sure you have `helm` installed, then simply run the following script: - -```bash -./scripts/install_logging.sh -``` - -To forward port to view Grafana dashboard: - -```bash -./scripts/connect_logging.sh -``` - -## Kubernetes (e.g. minikube) - -To start the server run: - -```bash -warnet -``` - -### Running large networks - -When running a large number of containers on a single host machine (i.e. with the Docker interface), the system may run out of various resources. +When running a large number of containers on a single host machine, the system may run out of various resources. We recommend setting the following values in /etc/sysctl.conf: ```sh @@ -65,7 +42,7 @@ Apply the settings by either restarting the host, or without restarting using: sudo sysctl -p ``` -In addition to these settings, you may need to increase the maximum number of permitted open files for the user running the docker daemon (usually root) in /etc/security/limits.conf. +In addition to these settings, you may need to increase the maximum number of permitted open files in /etc/security/limits.conf. This change is often not necessary though so we recommend trying your network without it first. The following command will apply it to a single shell session, and not persist it. @@ -75,19 +52,3 @@ Use as root before launching docker. # Increase the number of open files allowed per process to 4096 ulimit -n 4096 ``` - -If you are running docker as a service via systemd you can apply it by adding the following to the service file and restarting the service: - -```sh -# Add the following under the [Service] section of the unit file -LimitNOFILE=4096 -``` - -Reload the systemd configuration and restart the unit afterwards: - -``` -sudo systemctl daemon-reload -sudo systemctl restart docker -``` - -On Ubuntu this file is located at `/lib/systemd/system/docker.service` but you can find it using `sudo systemctl status docker`. diff --git a/resources/scripts/connect_logging.sh b/resources/scripts/connect_logging.sh index 98847581f..94edc35ae 100755 --- a/resources/scripts/connect_logging.sh +++ b/resources/scripts/connect_logging.sh @@ -1,8 +1,16 @@ #!/bin/bash -set -e +# NO `set -e` here so an error does not exit the script POD_NAME=$(kubectl get pods --namespace warnet-logging -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=loki-grafana" -o jsonpath="{.items[0].metadata.name}") echo "Go to http://localhost:3000" +echo "Grafana pod name: ${POD_NAME}" -kubectl --namespace warnet-logging port-forward "${POD_NAME}" 3000 \ No newline at end of file +while true; do + echo "Attempting to start Grafana port forwarding" + kubectl --namespace warnet-logging port-forward "${POD_NAME}" 3000 2>&1 + echo "Grafana port forwarding exited with status: $?" + sleep 5 +done; + +echo "warnet-logging port-forward exited" \ No newline at end of file