diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7141d5265..ddef42f1b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -38,7 +38,7 @@ jobs: - name: Build and push production RPC image uses: docker/build-push-action@v5 with: - file: src/templates/rpc/Dockerfile_rpc + file: resources/images/rpc/Dockerfile_prod platforms: linux/amd64,linux/arm64 context: . push: true @@ -50,7 +50,7 @@ jobs: if: github.ref == 'refs/heads/main' uses: docker/build-push-action@v5 with: - file: src/templates/rpc/Dockerfile_rpc_dev + file: resources/images/rpc/Dockerfile_dev platforms: linux/amd64,linux/arm64 context: . push: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 551d0ff81..348551aac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Build and export uses: docker/build-push-action@v5 with: - file: src/warnet/templates/rpc/Dockerfile_rpc + file: resources/images/rpc/Dockerfile_prod context: . tags: warnet/dev cache-from: type=gha @@ -74,13 +74,13 @@ jobs: uv pip install -e . echo "Contents of warnet-rpc-statefulset-dev.yaml being used:" - cat src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml + cat resources/manifests/warnet-rpc-statefulset-dev.yaml echo Setting up k8s - kubectl apply -f src/warnet/templates/rpc/namespace.yaml - kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml + kubectl apply -f resources/manifests/namespace.yaml + kubectl apply -f resources/manifests/rbac-config.yaml + kubectl apply -f resources/manifests/warnet-rpc-service.yaml + kubectl apply -f resources/manifests/warnet-rpc-statefulset-dev.yaml kubectl config set-context --current --namespace=warnet echo sleeping for 30s to give k8s time to boot diff --git a/MANIFEST.in b/MANIFEST.in index 73e895e97..e96af9d3b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,2 @@ -include src/warnet/schema/*.json -include src/warnet/templates/bitcoin.conf -include src/warnet/templates/fork_observer_config.toml -include src/warnet/templates/addrman_observer_config.toml -include src/warnet/templates/addrman.patch -include src/warnet/templates/isroutable.patch -include src/warnet/scripts/quick_start.sh -graft src/warnet/templates/k8s -graft src/warnet/templates/rpc -graft src/warnet/templates/grafana-provisioning -graft src/warnet/logging_config +include src/warnet/*.json +graft resources diff --git a/docs/developer-notes.md b/docs/developer-notes.md index 2fa07038a..47e22e812 100644 --- a/docs/developer-notes.md +++ b/docs/developer-notes.md @@ -13,9 +13,9 @@ To help with this a helper script is provided: [build-k8s-rpc.sh](../scripts/bui This script can be run in the following way: ```bash -DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 ./scripts/build-k8s-rpc.sh Dockerfile_rpc +DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 ./scripts/build-k8s-rpc.sh Dockerfile_prod ``` You can optionally specify `LATEST=1` to also include the `latest` tag on docker hub. -Once a new image has been pushed, it should be referenced in [warnet-rpc-statefulset.yaml](../src/warnet/templates/warnet-rpc-statefulset.yaml) in the `image` field. +Once a new image has been pushed, it should be referenced in manifests/warnet-rpc-statefulset.yaml in the `image` field. diff --git a/docs/graph.md b/docs/graph.md index 9caaf4633..fa7da6534 100644 --- a/docs/graph.md +++ b/docs/graph.md @@ -5,7 +5,7 @@ Warnet creates a Bitcoin network using a network topology from a [graphml](https Before any scenarios or RPC commands can be executed, a Warnet network must be started from a graph. See [warcli.md](warcli.md) for more details on these commands. -To start a network called `"warnet"` from the [default graph file](../src/graphs/default.graphml): +To start a network called `"warnet"` from the [default graph file](../graphs/default.graphml): ``` warcli network start ``` diff --git a/docs/running.md b/docs/running.md index 3e8773788..aea8e0641 100644 --- a/docs/running.md +++ b/docs/running.md @@ -9,44 +9,20 @@ a file `$XDG_STATE_HOME/warnet/warnet.log`, otherwise it will use `$HOME/.warnet ## Kubernetes -Deploy the resources in `src/warnet/templates/`, this sets up the correct permissions on the cluster (`rbac-config.yaml`) and deploys the warnet RPC server as a service + statefulset. - -This can be done with from inside the `src/warnet/templates/` directory by running: - -```bash -kubectl apply -f '*.yaml' -``` - -Once the RPC server comes up we need to forward the RPC port from the cluster. -This can be done with: - -```bash -kubectl port-forward svc/rpc 9276:9276 -``` - -This allows you to communicate with the RPC server using `warcli`. Developers -should check the [developer notes](developer-notes.md) to see how to -update the RPC server when developing on Kubernetes. - -Currently, while `warcli network down` will bring down the pods, the RPC server needs manual deletion. -This can be done using: - -```bash -kubectl delete statefulset -``` +// TODO ### Install logging infrastructure First make sure you have `helm` installed, then simply run the following script: ```bash -./src/warnet/templates/k8s/install_logging.sh +./scripts/install_logging.sh ``` To forward port to view Grafana dashboard: ```bash -./src/warnet/templates/k8s/connect_logging.sh +./scripts/connect_logging.sh ``` ## Kubernetes (e.g. minikube) diff --git a/justfile b/justfile index 62d3b64aa..fa53f8f5e 100644 --- a/justfile +++ b/justfile @@ -5,10 +5,10 @@ default: just --list cluster: - kubectl apply -f src/warnet/templates/rpc/namespace.yaml - kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-statefulset.yaml + kubectl apply -f resources/manifests/namespace.yaml + kubectl apply -f resources/manifests/rbac-config.yaml + kubectl apply -f resources/manifests/warnet-rpc-service.yaml + kubectl apply -f resources/manifests/warnet-rpc-statefulset.yaml # Setup and start the RPC in dev mode with minikube start: @@ -32,14 +32,14 @@ start: check_minikube # Build image in local registry and load into minikube - docker build -t warnet/dev -f src/warnet/templates/rpc/Dockerfile_rpc_dev src/warnet/templates/rpc --load + docker build -t warnet/dev -f resources/images/rpc/Dockerfile_dev resources/images/rpc --load minikube image load warnet/dev # Setup k8s - kubectl apply -f src/warnet/templates/rpc/namespace.yaml - kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml + kubectl apply -f resources/manifests/namespace.yaml + kubectl apply -f resources/manifests/rbac-config.yaml + kubectl apply -f resources/manifests/warnet-rpc-service.yaml + kubectl apply -f resources/manifests/warnet-rpc-statefulset-dev.yaml kubectl config set-context --current --namespace=warnet # Check for warnet-rpc container @@ -68,11 +68,11 @@ stop: # Setup and start the RPC in dev mode with Docker Desktop startd: - docker build -t warnet/dev -f src/warnet/templates/rpc/Dockerfile_rpc_dev src/warnet/templates/rpc --load - kubectl apply -f src/warnet/templates/rpc/namespace.yaml - kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml - kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml - sed 's?/mnt/src?'`PWD`'?g' src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml | kubectl apply -f - + docker build -t warnet/dev -f resources/images/rpc/Dockerfile_dev resources/images/rpc --load + kubectl apply -f resources/manifests/namespace.yaml + kubectl apply -f resources/manifests/rbac-config.yaml + kubectl apply -f resources/manifests/warnet-rpc-service.yaml + sed 's?/mnt/src?'`PWD`'?g' resources/manifests/warnet-rpc-statefulset-dev.yaml | kubectl apply -f - kubectl config set-context --current --namespace=warnet echo waiting for rpc to come online @@ -104,10 +104,10 @@ build branch tag registry=registry repo=repo build-args=build-args action=load: warcli image build --registry={{registry}} --repo={{repo}} --branch={{branch}} --arches="{{arches}}" --tag={{tag}} --build-args="{{build-args}}" --action={{action}} installlogging: - ./src/warnet/templates/k8s/install_logging.sh + resources/scripts/install_logging.sh connectlogging: - ./src/warnet/templates/k8s/connect_logging.sh + resources/scripts/connect_logging.sh # Format and lint all files lint: diff --git a/pyproject.toml b/pyproject.toml index 2e4b24432..729c797eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,11 @@ build-backend = "setuptools.build_meta" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} +[tool.setuptools] +include-package-data = true + [tool.setuptools.packages.find] -where = ["src"] +where = ["src", "resources"] [tool.ruff] extend-exclude = [ diff --git a/test/framework_tests/__init__.py b/resources/__init__.py similarity index 100% rename from test/framework_tests/__init__.py rename to resources/__init__.py diff --git a/resources/graphs/__init__.py b/resources/graphs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/warnet/graphs/default.graphml b/resources/graphs/default.graphml similarity index 100% rename from src/warnet/graphs/default.graphml rename to resources/graphs/default.graphml diff --git a/resources/images/__init__.py b/resources/images/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/warnet/templates/Dockerfile b/resources/images/bitcoin/Dockerfile similarity index 91% rename from src/warnet/templates/Dockerfile rename to resources/images/bitcoin/Dockerfile index 1d4e064de..2e4a7d1e2 100644 --- a/src/warnet/templates/Dockerfile +++ b/resources/images/bitcoin/Dockerfile @@ -22,8 +22,8 @@ RUN --mount=type=cache,target=/var/cache/apk \ sqlite-dev \ zeromq-dev -COPY src/warnet/templates/isroutable.patch /tmp/ -COPY src/warnet/templates/addrman.patch /tmp/ +COPY isroutable.patch /tmp/ +COPY addrman.patch /tmp/ # Clone and patch and build stage @@ -77,8 +77,7 @@ RUN --mount=type=cache,target=/var/cache/apk sed -i 's/http\:\/\/dl-cdn.alpineli su-exec COPY --from=build /opt/bitcoin /usr/local -COPY src/warnet/templates/entrypoint.sh /entrypoint.sh -COPY src/warnet/templates/tor/torrc /etc/tor/warnet-torr +COPY entrypoint.sh / VOLUME ["/home/bitcoin/.bitcoin"] EXPOSE 8332 8333 18332 18333 18443 18444 38333 38332 diff --git a/src/warnet/templates/addrman.patch b/resources/images/bitcoin/addrman.patch similarity index 100% rename from src/warnet/templates/addrman.patch rename to resources/images/bitcoin/addrman.patch diff --git a/src/warnet/templates/entrypoint.sh b/resources/images/bitcoin/entrypoint.sh similarity index 100% rename from src/warnet/templates/entrypoint.sh rename to resources/images/bitcoin/entrypoint.sh diff --git a/src/warnet/templates/isroutable.patch b/resources/images/bitcoin/isroutable.patch similarity index 100% rename from src/warnet/templates/isroutable.patch rename to resources/images/bitcoin/isroutable.patch diff --git a/src/warnet/templates/rpc/Dockerfile_rpc_dev b/resources/images/rpc/Dockerfile_dev similarity index 84% rename from src/warnet/templates/rpc/Dockerfile_rpc_dev rename to resources/images/rpc/Dockerfile_dev index cd7bd0ce2..48f348228 100644 --- a/src/warnet/templates/rpc/Dockerfile_rpc_dev +++ b/resources/images/rpc/Dockerfile_dev @@ -20,13 +20,10 @@ EXPOSE 9276 # we defer this to the entrypoint script for dev mode to enable hot-reloading. # Copy the entrypoint script into the container -COPY entrypoint.sh /usr/local/bin/ - -# Make the entrypoint script executable -RUN chmod +x /usr/local/bin/entrypoint.sh +COPY entrypoint.sh / # Set the entrypoint script to run when the container launches -ENTRYPOINT ["entrypoint.sh"] +ENTRYPOINT ["/entrypoint.sh"] # Default command CMD ["warnet", "--dev"] diff --git a/src/warnet/templates/rpc/Dockerfile_rpc b/resources/images/rpc/Dockerfile_prod similarity index 100% rename from src/warnet/templates/rpc/Dockerfile_rpc rename to resources/images/rpc/Dockerfile_prod diff --git a/src/warnet/templates/rpc/Dockerfile_rpc.dockerignore b/resources/images/rpc/Dockerfile_rpc.dockerignore similarity index 100% rename from src/warnet/templates/rpc/Dockerfile_rpc.dockerignore rename to resources/images/rpc/Dockerfile_rpc.dockerignore diff --git a/src/warnet/templates/rpc/entrypoint.sh b/resources/images/rpc/entrypoint.sh old mode 100644 new mode 100755 similarity index 100% rename from src/warnet/templates/rpc/entrypoint.sh rename to resources/images/rpc/entrypoint.sh diff --git a/src/warnet/templates/Dockerfile_sidecar b/resources/images/sidecar/Dockerfile similarity index 100% rename from src/warnet/templates/Dockerfile_sidecar rename to resources/images/sidecar/Dockerfile diff --git a/src/warnet/templates/tor/Dockerfile_tor_da b/resources/images/tor/Dockerfile_tor_da similarity index 100% rename from src/warnet/templates/tor/Dockerfile_tor_da rename to resources/images/tor/Dockerfile_tor_da diff --git a/src/warnet/templates/tor/Dockerfile_tor_relay b/resources/images/tor/Dockerfile_tor_relay similarity index 100% rename from src/warnet/templates/tor/Dockerfile_tor_relay rename to resources/images/tor/Dockerfile_tor_relay diff --git a/src/warnet/templates/tor/tor-entrypoint.sh b/resources/images/tor/tor-entrypoint.sh similarity index 100% rename from src/warnet/templates/tor/tor-entrypoint.sh rename to resources/images/tor/tor-entrypoint.sh diff --git a/src/warnet/templates/tor/tor-keys/authority_certificate b/resources/images/tor/tor-keys/authority_certificate similarity index 100% rename from src/warnet/templates/tor/tor-keys/authority_certificate rename to resources/images/tor/tor-keys/authority_certificate diff --git a/src/warnet/templates/tor/tor-keys/authority_identity_key b/resources/images/tor/tor-keys/authority_identity_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/authority_identity_key rename to resources/images/tor/tor-keys/authority_identity_key diff --git a/src/warnet/templates/tor/tor-keys/authority_signing_key b/resources/images/tor/tor-keys/authority_signing_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/authority_signing_key rename to resources/images/tor/tor-keys/authority_signing_key diff --git a/src/warnet/templates/tor/tor-keys/ed25519_master_id_public_key b/resources/images/tor/tor-keys/ed25519_master_id_public_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/ed25519_master_id_public_key rename to resources/images/tor/tor-keys/ed25519_master_id_public_key diff --git a/src/warnet/templates/tor/tor-keys/ed25519_master_id_secret_key b/resources/images/tor/tor-keys/ed25519_master_id_secret_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/ed25519_master_id_secret_key rename to resources/images/tor/tor-keys/ed25519_master_id_secret_key diff --git a/src/warnet/templates/tor/tor-keys/ed25519_signing_cert b/resources/images/tor/tor-keys/ed25519_signing_cert similarity index 100% rename from src/warnet/templates/tor/tor-keys/ed25519_signing_cert rename to resources/images/tor/tor-keys/ed25519_signing_cert diff --git a/src/warnet/templates/tor/tor-keys/ed25519_signing_secret_key b/resources/images/tor/tor-keys/ed25519_signing_secret_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/ed25519_signing_secret_key rename to resources/images/tor/tor-keys/ed25519_signing_secret_key diff --git a/src/warnet/templates/tor/tor-keys/secret_id_key b/resources/images/tor/tor-keys/secret_id_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/secret_id_key rename to resources/images/tor/tor-keys/secret_id_key diff --git a/src/warnet/templates/tor/tor-keys/secret_onion_key b/resources/images/tor/tor-keys/secret_onion_key similarity index 100% rename from src/warnet/templates/tor/tor-keys/secret_onion_key rename to resources/images/tor/tor-keys/secret_onion_key diff --git a/src/warnet/templates/tor/tor-keys/secret_onion_key_ntor b/resources/images/tor/tor-keys/secret_onion_key_ntor similarity index 100% rename from src/warnet/templates/tor/tor-keys/secret_onion_key_ntor rename to resources/images/tor/tor-keys/secret_onion_key_ntor diff --git a/src/warnet/templates/tor/torrc b/resources/images/tor/torrc similarity index 100% rename from src/warnet/templates/tor/torrc rename to resources/images/tor/torrc diff --git a/src/warnet/templates/tor/torrc.da b/resources/images/tor/torrc.da similarity index 100% rename from src/warnet/templates/tor/torrc.da rename to resources/images/tor/torrc.da diff --git a/src/warnet/templates/tor/torrc.relay b/resources/images/tor/torrc.relay similarity index 100% rename from src/warnet/templates/tor/torrc.relay rename to resources/images/tor/torrc.relay diff --git a/resources/manifests/__init__.py b/resources/manifests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/warnet/templates/k8s/grafana/values.yaml b/resources/manifests/grafana_values.yaml similarity index 100% rename from src/warnet/templates/k8s/grafana/values.yaml rename to resources/manifests/grafana_values.yaml diff --git a/src/warnet/templates/k8s/loki/values.yaml b/resources/manifests/loki_values.yaml similarity index 100% rename from src/warnet/templates/k8s/loki/values.yaml rename to resources/manifests/loki_values.yaml diff --git a/src/warnet/templates/rpc/namespace.yaml b/resources/manifests/namespace.yaml similarity index 100% rename from src/warnet/templates/rpc/namespace.yaml rename to resources/manifests/namespace.yaml diff --git a/src/warnet/templates/rpc/rbac-config.yaml b/resources/manifests/rbac-config.yaml similarity index 100% rename from src/warnet/templates/rpc/rbac-config.yaml rename to resources/manifests/rbac-config.yaml diff --git a/src/warnet/templates/rpc/warnet-rpc-service.yaml b/resources/manifests/warnet-rpc-service.yaml similarity index 100% rename from src/warnet/templates/rpc/warnet-rpc-service.yaml rename to resources/manifests/warnet-rpc-service.yaml diff --git a/src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml b/resources/manifests/warnet-rpc-statefulset-dev.yaml similarity index 83% rename from src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml rename to resources/manifests/warnet-rpc-statefulset-dev.yaml index 5bee597ff..91692e35a 100644 --- a/src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml +++ b/resources/manifests/warnet-rpc-statefulset-dev.yaml @@ -26,7 +26,15 @@ spec: livenessProbe: # fail (restart) the pod if we can't find pid of warnet exec: - command: [ "/bin/bash", "-c", "/root/warnet/src/warnet/templates/rpc/livenessProbe.sh" ] + command: + - /bin/sh + - -c + - | + if pgrep -f warnet > /dev/null; then + exit 0 + else + exit 1 + fi initialDelaySeconds: 20 periodSeconds: 5 failureThreshold: 3 diff --git a/src/warnet/templates/rpc/warnet-rpc-statefulset.yaml b/resources/manifests/warnet-rpc-statefulset.yaml similarity index 100% rename from src/warnet/templates/rpc/warnet-rpc-statefulset.yaml rename to resources/manifests/warnet-rpc-statefulset.yaml diff --git a/resources/scripts/__init__.py b/resources/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/apidocs.py b/resources/scripts/apidocs.py similarity index 100% rename from scripts/apidocs.py rename to resources/scripts/apidocs.py diff --git a/scripts/build-k8s-rpc.sh b/resources/scripts/build-k8s-rpc.sh similarity index 96% rename from scripts/build-k8s-rpc.sh rename to resources/scripts/build-k8s-rpc.sh index a261162b8..a61291291 100755 --- a/scripts/build-k8s-rpc.sh +++ b/resources/scripts/build-k8s-rpc.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Run with e.g.: -# $ DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 LATEST=1 ./scripts/build-k8s-rpc.sh Dockerfile_rpc +# $ DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 LATEST=1 ./scripts/build-k8s-rpc.sh Dockerfile_prod # Fail on any step set -ex diff --git a/src/warnet/templates/k8s/connect_logging.sh b/resources/scripts/connect_logging.sh similarity index 100% rename from src/warnet/templates/k8s/connect_logging.sh rename to resources/scripts/connect_logging.sh diff --git a/scripts/graphdocs.py b/resources/scripts/graphdocs.py similarity index 100% rename from scripts/graphdocs.py rename to resources/scripts/graphdocs.py diff --git a/src/warnet/templates/k8s/install_logging.sh b/resources/scripts/install_logging.sh similarity index 65% rename from src/warnet/templates/k8s/install_logging.sh rename to resources/scripts/install_logging.sh index e3a253680..15fd1ab1e 100755 --- a/src/warnet/templates/k8s/install_logging.sh +++ b/resources/scripts/install_logging.sh @@ -1,13 +1,14 @@ #!/bin/bash set -e -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +MANIFESTS_DIR=$( cd -- "$THIS_DIR/../manifests" &> /dev/null && pwd ) helm repo add grafana https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update -helm upgrade --install --namespace warnet-logging --create-namespace --values "${SCRIPT_DIR}/loki/values.yaml" loki grafana/loki --version 5.47.2 +helm upgrade --install --namespace warnet-logging --create-namespace --values "${MANIFESTS_DIR}/loki_values.yaml" loki grafana/loki --version 5.47.2 helm upgrade --install --namespace warnet-logging promtail grafana/promtail helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false -helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${SCRIPT_DIR}/grafana/values.yaml" \ No newline at end of file +helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${MANIFESTS_DIR}/grafana_values.yaml" \ No newline at end of file diff --git a/src/warnet/scripts/quick_start.sh b/resources/scripts/quick_start.sh similarity index 100% rename from src/warnet/scripts/quick_start.sh rename to resources/scripts/quick_start.sh diff --git a/src/warnet/__init__.py b/src/warnet/__init__.py index e69de29bb..db25872f7 100644 --- a/src/warnet/__init__.py +++ b/src/warnet/__init__.py @@ -0,0 +1,3 @@ +from importlib.resources import files + +SRC_DIR = files("warnet") diff --git a/src/warnet/cli/cluster.py b/src/warnet/cli/cluster.py index da263a35d..74a9f99a5 100644 --- a/src/warnet/cli/cluster.py +++ b/src/warnet/cli/cluster.py @@ -1,10 +1,13 @@ import os import subprocess import sys -from importlib import resources +from importlib.resources import files import click +MANIFEST_PATH = files("manifests") +RPC_PATH = files("images").joinpath("rpc") + @click.group(name="cluster", chain=True) def cluster(): @@ -52,67 +55,65 @@ def run_command(command, stream_output=False): @cluster.command() def minikube_setup(): """Setup minikube for use with Warnet""" - with resources.path("warnet.templates", "") as template_path: - script_content = f""" - #!/usr/bin/env bash - set -euxo pipefail + script_content = f""" + #!/usr/bin/env bash + set -euxo pipefail - # Function to check if minikube is running - check_minikube() {{ - minikube status | grep -q "Running" && echo "Minikube is already running" || minikube start --memory=max --cpus=max --mount --mount-string="$PWD:/mnt/src" - }} + # Function to check if minikube is running + check_minikube() {{ + minikube status | grep -q "Running" && echo "Minikube is already running" || minikube start --memory=4000mb --cpus=4 --mount --mount-string="$PWD:/mnt/src" + }} - # Check minikube status - check_minikube + # Check minikube status + check_minikube - # Build image in local registry and load into minikube - docker build -t warnet/dev -f {template_path}/rpc/Dockerfile_rpc_dev {template_path}/rpc --load - minikube image load warnet/dev - """ + # Build image in local registry and load into minikube + docker build -t warnet/dev -f {RPC_PATH}/Dockerfile_dev {RPC_PATH} --load + minikube image load warnet/dev + """ - run_command(script_content, stream_output=True) + run_command(script_content, stream_output=True) @cluster.command() def deploy(): """Setup Warnet using the current kubectl-configured cluster""" - with resources.path("warnet.templates", "") as template_path: - script_content = f""" - #!/usr/bin/env bash - set -euxo pipefail - - # Function to check if warnet-rpc container is already running - check_warnet_rpc() {{ - if kubectl get pods --all-namespaces | grep -q "bitcoindevproject/warnet-rpc"; then - echo "warnet-rpc already running in minikube" - exit 1 - fi - }} - - # Setup K8s - kubectl apply -f {template_path}/rpc/namespace.yaml - kubectl apply -f {template_path}/rpc/rbac-config.yaml - kubectl apply -f {template_path}/rpc/warnet-rpc-service.yaml - kubectl apply -f {template_path}/rpc/warnet-rpc-statefulset-dev.yaml - kubectl config set-context --current --namespace=warnet - - # Check for warnet-rpc container - check_warnet_rpc - - until kubectl get pod rpc-0 --namespace=warnet; do - echo "Waiting for server to find pod rpc-0..." - sleep 4 - done - - echo "⏲️ This could take a minute or so." - kubectl wait --for=condition=Ready --timeout=2m pod rpc-0 - - echo Done... - """ - - res = run_command(script_content, stream_output=True) - if res: - _port_start_internal() + script_content = f""" + #!/usr/bin/env bash + set -euxo pipefail + + # Function to check if warnet-rpc container is already running + check_warnet_rpc() {{ + if kubectl get pods --all-namespaces | grep -q "bitcoindevproject/warnet-rpc"; then + echo "warnet-rpc already running in minikube" + exit 1 + fi + }} + + # Setup K8s + kubectl apply -f {MANIFEST_PATH}/namespace.yaml + kubectl apply -f {MANIFEST_PATH}/rbac-config.yaml + kubectl apply -f {MANIFEST_PATH}/warnet-rpc-service.yaml + kubectl apply -f {MANIFEST_PATH}/warnet-rpc-statefulset-dev.yaml + kubectl config set-context --current --namespace=warnet + + # Check for warnet-rpc container + check_warnet_rpc + + until kubectl get pod rpc-0 --namespace=warnet; do + echo "Waiting for server to find pod rpc-0..." + sleep 4 + done + + echo "⏲️ This could take a minute or so." + kubectl wait --for=condition=Ready --timeout=2m pod rpc-0 + + echo Done... + """ + + res = run_command(script_content, stream_output=True) + if res: + _port_start_internal() @cluster.command() diff --git a/src/warnet/cli/image.py b/src/warnet/cli/image.py index 1fd1d2e13..6965838b6 100644 --- a/src/warnet/cli/image.py +++ b/src/warnet/cli/image.py @@ -17,8 +17,8 @@ def image(): @click.option("--tag", required=True, type=str) @click.option("--build-args", required=False, type=str) @click.option("--arches", required=False, type=str) -@click.option("--action", required=False, type=str) -def build(repo, commit_sha, registry, tag, build_args, arches, action="load"): +@click.option("--action", required=False, type=str, default="load") +def build(repo, commit_sha, registry, tag, build_args, arches, action): """ Build bitcoind and bitcoin-cli from at as :. Optionally deploy to remote registry using --action=push, otherwise image is loaded to local registry. diff --git a/src/warnet/cli/image_build.py b/src/warnet/cli/image_build.py index 151400e90..ace12a78d 100644 --- a/src/warnet/cli/image_build.py +++ b/src/warnet/cli/image_build.py @@ -1,8 +1,10 @@ -import importlib.resources import subprocess +from importlib.resources import files ARCHES = ["amd64", "arm64", "armhf"] +dockerfile_path = files("images.bitcoin").joinpath("Dockerfile") + def run_command(command): try: @@ -19,7 +21,7 @@ def build_image( tag: str, build_args: str, arches: str, - action: str = "load", + action: str, ): if not build_args: build_args = '"--disable-tests --without-gui --disable-bench --disable-fuzz-binary --enable-suppress-external-warnings "' @@ -44,15 +46,6 @@ def build_image( print(f"{build_args=:}") print(f"{build_arches=:}") - # Use importlib.resources to check if the templates directory exists - template_package = "warnet.templates" - with importlib.resources.path(template_package, "Dockerfile") as dockerfile_path: - dockerfile_dir = dockerfile_path.parent - if not dockerfile_dir.is_dir(): - print("Directory src/warnet/templates does not exist.") - print("Please run this script from the project root.") - return False - # Setup buildkit builder_name = "bitcoind-builder" create_builder_cmd = f"docker buildx create --name {builder_name} --use" @@ -77,7 +70,8 @@ def build_image( f" --build-arg COMMIT_SHA={commit_sha}" f" --build-arg BUILD_ARGS={build_args}" f" --tag {image_full_name}" - f" --file {dockerfile_path} ." + f" --file {dockerfile_path}" + f" {dockerfile_path.parent}" f" --{action}" ) print(f"Using {build_command=}") diff --git a/src/warnet/cli/main.py b/src/warnet/cli/main.py index 8aa112d10..6fd941dbb 100644 --- a/src/warnet/cli/main.py +++ b/src/warnet/cli/main.py @@ -1,7 +1,3 @@ -import os -import subprocess -from importlib import resources - import click from rich import print as richprint @@ -64,35 +60,5 @@ def help_command(ctx, commands): cli.add_command(help_command) -@cli.command() -def setup(): - """Run the Warnet quick start setup script""" - try: - with resources.path("warnet.scripts", "quick_start.sh") as script_path: - process = subprocess.Popen( - ["/bin/bash", str(script_path)], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - # This preserves colours from grant's lovely script! - env=dict(os.environ, TERM="xterm-256color"), - ) - - for line in iter(process.stdout.readline, ""): - print(line, end="", flush=True) - - process.stdout.close() - return_code = process.wait() - - if return_code != 0: - print(f"Quick start script failed with return code {return_code}") - return False - return True - - except Exception as e: - print(f"An error occurred while running the quick start script: {e}") - return False - - if __name__ == "__main__": cli() diff --git a/src/warnet/cli/network.py b/src/warnet/cli/network.py index e89e0e496..29039420b 100644 --- a/src/warnet/cli/network.py +++ b/src/warnet/cli/network.py @@ -1,16 +1,17 @@ import base64 # noqa: I001 +import json from pathlib import Path +from importlib.resources import files import click -import json from rich import print from rich.console import Console from rich.table import Table + from .rpc import rpc_call # noqa: I001 -from warnet.graphs import GRAPHS # noqa: I001 -DEFAULT_GRAPH_FILE = GRAPHS / "default.graphml" +DEFAULT_GRAPH_FILE = files("graphs").joinpath("default.graphml") def print_repr(wn: dict) -> None: diff --git a/src/warnet/schema/graph_schema.json b/src/warnet/graph_schema.json similarity index 100% rename from src/warnet/schema/graph_schema.json rename to src/warnet/graph_schema.json diff --git a/src/warnet/graphs/__init__.py b/src/warnet/graphs/__init__.py deleted file mode 100644 index 89c70da95..000000000 --- a/src/warnet/graphs/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pathlib import Path - -GRAPHS = Path(__file__).parent diff --git a/src/warnet/logging_config/config.json b/src/warnet/logging_config.json similarity index 100% rename from src/warnet/logging_config/config.json rename to src/warnet/logging_config.json diff --git a/src/warnet/schema/__init__.py b/src/warnet/schema/__init__.py deleted file mode 100644 index f09775abe..000000000 --- a/src/warnet/schema/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pathlib import Path - -SCHEMA = Path(__file__).parent diff --git a/src/warnet/server.py b/src/warnet/server.py index ef6345c6b..13066482c 100644 --- a/src/warnet/server.py +++ b/src/warnet/server.py @@ -1,7 +1,6 @@ import argparse import base64 import importlib -import importlib.resources as pkg_resources import io import json import logging @@ -23,6 +22,7 @@ from flask import Flask, jsonify, request from flask_jsonrpc.app import JSONRPC from flask_jsonrpc.exceptions import ServerError +from warnet import SRC_DIR from .services import ServiceType from .utils import gen_config_dir @@ -73,7 +73,7 @@ def __init__(self): def setup_global_exception_handler(self): """ - Use flask to log traceback of unhandled excpetions + Use flask to log traceback of unhandled exceptions """ @self.app.errorhandler(Exception) @@ -96,7 +96,7 @@ def healthy(self): def setup_logging(self): os.makedirs(os.path.dirname(self.log_file_path), exist_ok=True) - with pkg_resources.open_text("warnet.logging_config", "config.json") as f: + with open(SRC_DIR / "logging_config.json") as f: logging_config = json.load(f) logging_config["handlers"]["file"]["filename"] = str(self.log_file_path) logging.config.dictConfig(logging_config) diff --git a/src/warnet/templates/__init__.py b/src/warnet/templates/__init__.py deleted file mode 100644 index 31005bd2a..000000000 --- a/src/warnet/templates/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pathlib import Path - -TEMPLATES = Path(__file__).parent diff --git a/src/warnet/templates/addrman_observer_config.toml b/src/warnet/templates/addrman_observer_config.toml deleted file mode 100644 index 4bfcdb79b..000000000 --- a/src/warnet/templates/addrman_observer_config.toml +++ /dev/null @@ -1,8 +0,0 @@ -# addrman-observer base configuration file - -# path to the location of the static www files -www_path = "./www" - -# webserver listen address -address = "0.0.0.0:3882" - diff --git a/src/warnet/templates/archive/fluent.conf b/src/warnet/templates/archive/fluent.conf deleted file mode 100644 index c545b7412..000000000 --- a/src/warnet/templates/archive/fluent.conf +++ /dev/null @@ -1,28 +0,0 @@ - - @type forward - port 24224 - bind 0.0.0.0 - - - - @type record_transformer - enable_ruby - - message ${record['container_name'].gsub(/^\//, '')}: ${record['log']} - - - - - @type stdout - - @type single_value - message_key message - - - - - @type http - port 24220 - bind 0.0.0.0 - - diff --git a/src/warnet/templates/docker_entrypoint.sh b/src/warnet/templates/docker_entrypoint.sh deleted file mode 100755 index 9045394f6..000000000 --- a/src/warnet/templates/docker_entrypoint.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e - -# Add bitcoin user to tor group to read the auth cookie -usermod -a -G debian-tor bitcoin -echo "Address $(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)" >> /etc/tor/torrc -mkdir -p /home/debian-tor/.tor/keys -chown -R debian-tor:debian-tor /home/debian-tor -# Start tor in the background -gosu debian-tor tor & - -if [ -n "${UID+x}" ] && [ "${UID}" != "0" ]; then - usermod -u "$UID" bitcoin -fi - -if [ -n "${GID+x}" ] && [ "${GID}" != "0" ]; then - groupmod -g "$GID" bitcoin -fi - -echo "$0: assuming uid:gid for bitcoin:bitcoin of $(id -u bitcoin):$(id -g bitcoin)" - -if [ "$(echo "$1" | cut -c1)" = "-" ]; then - echo "$0: assuming arguments for bitcoind" - - set -- bitcoind "$@" -fi - -if [ "$(echo "$1" | cut -c1)" = "-" ] || [ "$1" = "bitcoind" ]; then - mkdir -p "$BITCOIN_DATA" - chmod 700 "$BITCOIN_DATA" - # Fix permissions for home dir. - chown -R bitcoin:bitcoin "$(getent passwd bitcoin | cut -d: -f6)" - # Fix permissions for bitcoin data dir. - chown -R bitcoin:bitcoin "$BITCOIN_DATA" - - echo "$0: setting data directory to $BITCOIN_DATA" - - set -- "$@" -datadir="$BITCOIN_DATA" -fi - -if [ "$1" = "bitcoind" ] || [ "$1" = "bitcoin-cli" ] || [ "$1" = "bitcoin-tx" ]; then - echo - exec gosu bitcoin "$@" -fi - -echo -exec "$@" diff --git a/src/warnet/templates/fork_observer_config.toml b/src/warnet/templates/fork_observer_config.toml deleted file mode 100644 index 05c6a81c1..000000000 --- a/src/warnet/templates/fork_observer_config.toml +++ /dev/null @@ -1,31 +0,0 @@ -# fork-observer base configuration file - -# Database path of the key value store. Will be created if non-existing. -database_path = "db" - -# path to the location of the static www files -www_path = "./www" - -# Interval for checking for new blocks -query_interval = 60 - -# Webserver listen address -address = "0.0.0.0:2323" - -# Custom footer for the site. -footer_html = """ -
-
- Warnet fork-observer -
-
- """ - -[[networks]] -id = 0xDEADBE -name = "Warnet" -description = "A custom regtest network" -min_fork_height = 0 -max_interesting_heights = 5 - - diff --git a/src/warnet/templates/grafana-provisioning/dashboards/10619_rev1.json b/src/warnet/templates/grafana-provisioning/dashboards/10619_rev1.json deleted file mode 100644 index f03eb305c..000000000 --- a/src/warnet/templates/grafana-provisioning/dashboards/10619_rev1.json +++ /dev/null @@ -1,1782 +0,0 @@ -{ - "__inputs": [ - { - "name": "prometheusdatasource", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.4.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "A simple overview of the most important Docker host and container metrics. (cAdvisor/Prometheus)", - "editable": true, - "gnetId": 10619, - "graphTooltip": 1, - "id": null, - "iteration": 1564137194972, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "prometheusdatasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 22, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "bucketAggs": [ - { - "field": "@timestamp", - "id": "2", - "settings": { - "interval": "auto", - "min_doc_count": 0, - "trimEdges": 0 - }, - "type": "date_histogram" - } - ], - "expr": "count(container_last_seen{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})", - "format": "time_series", - "intervalFactor": 1, - "metrics": [ - { - "field": "select field", - "id": "1", - "type": "count" - } - ], - "query": "count(container_last_seen{instance=~\"$node:$port\",job=~\"$job\",image!=\"\"})", - "refId": "A", - "timeField": "@timestamp" - } - ], - "thresholds": "", - "title": "Running containers", - "transparent": true, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": { - "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}": "#BA43A9" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 6, - "w": 8, - "x": 4, - "y": 0 - }, - "id": 5, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_system_seconds_total[1m]))", - "hide": true, - "intervalFactor": 2, - "legendFormat": "a", - "refId": "B", - "step": 120 - }, - { - "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m]))", - "hide": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "nur container", - "refId": "F", - "step": 10 - }, - { - "expr": "sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m]))", - "hide": true, - "interval": "", - "intervalFactor": 2, - "legendFormat": "nur docker host", - "metric": "", - "refId": "A", - "step": 20 - }, - { - "expr": "sum(rate(process_cpu_seconds_total[$interval])) * 100", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "host", - "metric": "", - "refId": "C", - "step": 4 - }, - { - "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m])) + sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m])) + sum(rate(process_cpu_seconds_total[1m]))", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "D", - "step": 120 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage on Node", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": false, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": "", - "logBase": 1, - "max": 120, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Belegete Festplatte": "#BF1B00", - "Free Disk Space": "#7EB26D", - "Used Disk Space": "#BF1B00", - "{}": "#BF1B00" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 4, - "grid": {}, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 0 - }, - "id": 13, - "isNew": true, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "node_filesystem_avail_bytes{fstype=\"xfs\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{mountpoint}}", - "refId": "A", - "step": 4 - }, - { - "expr": "node_filesystem_size_bytes{fstype=\"xfs\"} - node_filesystem_avail_bytes{fstype=\"xfs\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Used Disk Space", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Free and Used Disk Space on Node", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": false, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Available Memory": "#7EB26D", - "Unavailable Memory": "#BF1B00" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 4, - "grid": {}, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 0 - }, - "id": 20, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 3, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "container_memory_rss{name=~\".+\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "D", - "step": 30 - }, - { - "expr": "sum(container_memory_rss{name=~\".+\"})", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "A", - "step": 20 - }, - { - "expr": "container_memory_usage_bytes{name=~\".+\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "B", - "step": 20 - }, - { - "expr": "container_memory_rss{id=\"/\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "C", - "step": 30 - }, - { - "expr": "sum(container_memory_rss)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "E", - "step": 30 - }, - { - "expr": "node_memory_Buffers", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "node_memory_Dirty", - "refId": "N", - "step": 30 - }, - { - "expr": "node_memory_MemFree", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "F", - "step": 30 - }, - { - "expr": "node_memory_MemAvailable", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Available Memory", - "refId": "H", - "step": 4 - }, - { - "expr": "node_memory_MemTotal - node_memory_MemAvailable", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Unavailable Memory", - "refId": "G", - "step": 4 - }, - { - "expr": "node_memory_Inactive", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "I", - "step": 30 - }, - { - "expr": "node_memory_KernelStack", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "J", - "step": 30 - }, - { - "expr": "node_memory_Active", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "K", - "step": 30 - }, - { - "expr": "node_memory_MemTotal - (node_memory_Active + node_memory_MemFree + node_memory_Inactive)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "Unknown", - "refId": "L", - "step": 40 - }, - { - "expr": "node_memory_MemFree + node_memory_Inactive ", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "M", - "step": 30 - }, - { - "expr": "container_memory_rss{name=~\".+\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "refId": "O", - "step": 30 - }, - { - "expr": "node_memory_Inactive + node_memory_MemFree + node_memory_MemAvailable", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "P", - "step": 40 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Available Memory on Node", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": false, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": 4200000000, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 4, - "w": 2, - "x": 20, - "y": 0 - }, - "id": 3, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(node_disk_bytes_read[$interval])) by (device)", - "intervalFactor": 2, - "legendFormat": "OUT on /{{device}}", - "metric": "node_disk_bytes_read", - "refId": "A", - "step": 4 - }, - { - "expr": "sum(rate(node_disk_bytes_written[$interval])) by (device)", - "intervalFactor": 2, - "legendFormat": "IN on /{{device}}", - "metric": "", - "refId": "B", - "step": 4 - }, - { - "expr": "", - "intervalFactor": 2, - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk I/O on Node", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": false, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "node_load15": "#CCA300" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 0, - "grid": {}, - "gridPos": { - "h": 8, - "w": 10, - "x": 12, - "y": 4 - }, - "id": 4, - "isNew": true, - "legend": { - "avg": false, - "current": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "{__name__=~\"^node_load.*\"}", - "intervalFactor": 2, - "legendFormat": "{{__name__}}", - "metric": "node", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "System Load on Node", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "SENT": "#BF1B00" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 5, - "grid": {}, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 19, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 1, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\"}[$interval])) by (id)", - "intervalFactor": 2, - "legendFormat": "RECEIVED", - "refId": "A", - "step": 4 - }, - { - "expr": "- sum(rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])) by (id)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "SENT", - "refId": "B", - "step": 4 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network Traffic on Node", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": false, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 5, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 1, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\".+\"}[$interval])) by (name) * 100", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "container_cp", - "refId": "F", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage per Container (Stacked)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": "", - "logBase": 1, - "max": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 12 - }, - "id": 10, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_rss{name=~\".+\"}) by (name)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "container_memory_usage_bytes{name=~\".+\"}", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage per Container (Stacked)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 9, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Sent Network Traffic per Container", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 10, - "max": 8, - "min": 0, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 3, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 11, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "container_memory_rss{name=~\".+\"}", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 20 - }, - { - "expr": "container_memory_usage_bytes{name=~\".+\"}", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "B", - "step": 20 - }, - { - "expr": "sum(container_memory_cache{name=~\".+\"}) by (name)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "C", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Cached Memory per Container (Stacked)", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 8, - "isNew": true, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_network_receive_bytes_total{name=~\".+\"}[$interval])) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "A", - "step": 2 - }, - { - "expr": "- rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Received Network Traffic per Container", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "columns": [ - { - "text": "Avg", - "value": "avg" - } - ], - "datasource": "prometheusdatasource", - "editable": true, - "error": false, - "fontSize": "100%", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 24 - }, - "hideTimeOverride": false, - "id": 18, - "isNew": true, - "links": [], - "pageSize": 100, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "cAdvisor Version: {{cadvisorVersion}}", - "refId": "A", - "step": 2 - }, - { - "expr": "prometheus_build_info", - "intervalFactor": 2, - "legendFormat": "Prometheus Version: {{version}}", - "refId": "B", - "step": 2 - }, - { - "expr": "node_exporter_build_info", - "intervalFactor": 2, - "legendFormat": "Node-Exporter Version: {{version}}", - "refId": "C", - "step": 2 - }, - { - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "Docker Version: {{dockerVersion}}", - "refId": "D", - "step": 2 - }, - { - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "Host OS Version: {{osVersion}}", - "refId": "E", - "step": 2 - }, - { - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "Host Kernel Version: {{kernelVersion}}", - "refId": "F", - "step": 2 - } - ], - "timeFrom": null, - "timeShift": null, - "title": "", - "transform": "timeseries_aggregations", - "type": "table" - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 50, - "auto_min": "50s", - "current": { - "text": "auto", - "value": "$__auto_interval_interval" - }, - "datasource": null, - "hide": 0, - "includeAll": false, - "label": "Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "2m", - "value": "2m" - }, - { - "selected": false, - "text": "3m", - "value": "3m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "7m", - "value": "7m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "allValue": null, - "current": {}, - "datasource": "prometheusdatasource", - "definition": "label_values(container_cpu_user_seconds_total, job)", - "hide": 0, - "includeAll": false, - "label": "Job", - "multi": false, - "name": "job", - "options": [], - "query": "label_values(container_cpu_user_seconds_total, job)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "All", - "current": {}, - "datasource": "prometheusdatasource", - "definition": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)", - "hide": 0, - "includeAll": true, - "label": "Host:", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)", - "refresh": 1, - "regex": "/([^:]+):.*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "prometheusdatasource", - "definition": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)", - "hide": 0, - "includeAll": false, - "label": "Port", - "multi": false, - "name": "port", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)", - "refresh": 1, - "regex": "/[^:]+:(.*)/", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "All", - "current": {}, - "datasource": "prometheusdatasource", - "definition": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.220.32:8080|10.115.220.33:8080\", name!=\"cadvisor\"}, name)", - "hide": 0, - "includeAll": true, - "label": "Prod", - "multi": true, - "name": "Prod", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.220.32:8080|10.115.220.33:8080\", name!=\"cadvisor\"}, name)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "apps.prod", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "All", - "current": {}, - "datasource": "prometheusdatasource", - "definition": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.222.21:8080|10.115.222.22:8080\", name!=\"cadvisor\"}, name)", - "hide": 0, - "includeAll": true, - "label": "NonProd", - "multi": true, - "name": "NonProd", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.222.21:8080|10.115.222.22:8080\", name!=\"cadvisor\"}, name)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Docker Container & Host Metrics", - "uid": "4dMaCsRZz", - "version": 27 -} diff --git a/src/warnet/templates/grafana-provisioning/dashboards/10619_rev2.json b/src/warnet/templates/grafana-provisioning/dashboards/10619_rev2.json deleted file mode 100644 index 0120c1dc9..000000000 --- a/src/warnet/templates/grafana-provisioning/dashboards/10619_rev2.json +++ /dev/null @@ -1,1260 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "A simple overview of the most important tank container metrics. (cAdvisor/Prometheus)", - "editable": true, - "fiscalYearStartMonth": 0, - "gnetId": 10619, - "graphTooltip": 1, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"$tanks\"}[$interval])) by (name) * 100", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "metric": "container_cp", - "range": true, - "refId": "F", - "step": 2 - } - ], - "title": "CPU Usage per Container", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 9, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(container_network_transmit_bytes_total{name=~\"$tanks\"}) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - } - ], - "title": "Total Sent Network Traffic per Container", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 8, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(container_network_receive_bytes_total{name=~\"$tanks\"}) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "- rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "B", - "step": 10 - } - ], - "title": "Total Received Network Traffic per Container", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 19, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_transmit_bytes_total{name=~\"$tanks\"}[$interval])) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "", - "refId": "B", - "step": 10 - } - ], - "title": "Sent Network Traffic per Container (rate)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 20, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(rate(container_network_receive_bytes_total{name=~\"$tanks\"}[$interval])) by (name)", - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "- rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "refId": "B", - "step": 10 - } - ], - "title": "Received Network Traffic per Container (rate)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 51 - }, - "id": 10, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(container_memory_rss{name=~\"$tanks\"}) by (name)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "A", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "container_memory_usage_bytes{name=~\"$tanks\"}", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "B", - "step": 240 - } - ], - "title": "Memory Usage per Container", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 24, - "x": 0, - "y": 63 - }, - "id": 11, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "container_memory_rss{name=~\"$tanks\"}", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "A", - "step": 20 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "container_memory_usage_bytes{name=~\"$tanks\"}", - "hide": true, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "B", - "step": 20 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editorMode": "code", - "expr": "sum(container_memory_cache{name=~\"$tanks\"}) by (name)", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{name}}", - "range": true, - "refId": "C", - "step": 2 - } - ], - "title": "Cached Memory per Container", - "type": "timeseries" - }, - { - "columns": [ - { - "text": "Avg", - "value": "avg" - } - ], - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "editable": true, - "error": false, - "fontSize": "100%", - "gridPos": { - "h": 5, - "w": 5, - "x": 19, - "y": 75 - }, - "hideTimeOverride": false, - "id": 18, - "links": [], - "pageSize": 100, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "align": "auto", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "align": "auto", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "cAdvisor Version: {{cadvisorVersion}}", - "refId": "A", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "prometheus_build_info", - "intervalFactor": 2, - "legendFormat": "Prometheus Version: {{version}}", - "refId": "B", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "node_exporter_build_info", - "intervalFactor": 2, - "legendFormat": "Node-Exporter Version: {{version}}", - "refId": "C", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "Docker Version: {{dockerVersion}}", - "refId": "D", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "Host OS Version: {{osVersion}}", - "refId": "E", - "step": 2 - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "expr": "cadvisor_version_info", - "intervalFactor": 2, - "legendFormat": "Host Kernel Version: {{kernelVersion}}", - "refId": "F", - "step": 2 - } - ], - "transform": "timeseries_aggregations", - "type": "table-old" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "tags": [], - "templating": { - "list": [ - { - "auto": true, - "auto_count": 50, - "auto_min": "50s", - "current": { - "selected": false, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "hide": 0, - "includeAll": false, - "label": "Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "2m", - "value": "2m" - }, - { - "selected": false, - "text": "3m", - "value": "3m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "7m", - "value": "7m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - }, - { - "current": { - "selected": false, - "text": "cadvisor", - "value": "cadvisor" - }, - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "definition": "label_values(container_cpu_user_seconds_total, job)", - "hide": 0, - "includeAll": false, - "label": "Job", - "multi": false, - "name": "job", - "options": [], - "query": "label_values(container_cpu_user_seconds_total, job)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "All", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "definition": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)", - "hide": 0, - "includeAll": true, - "label": "Host:", - "multi": true, - "name": "node", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{job=~\"$job\"}, instance)", - "refresh": 1, - "regex": "/([^:]+):.*/", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "isNone": true, - "selected": false, - "text": "None", - "value": "" - }, - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "definition": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)", - "hide": 0, - "includeAll": false, - "label": "Port", - "multi": false, - "name": "port", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{instance=~\"$node:(.*)\"}, instance)", - "refresh": 1, - "regex": "/[^:]+:(.*)/", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "All", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "definition": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.220.32:8080|10.115.220.33:8080\", name!=\"cadvisor\"}, name)", - "hide": 0, - "includeAll": true, - "label": "Prod", - "multi": true, - "name": "Prod", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.220.32:8080|10.115.220.33:8080\", name!=\"cadvisor\"}, name)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": "All", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "definition": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.222.21:8080|10.115.222.22:8080\", name!=\"cadvisor\"}, name)", - "hide": 0, - "includeAll": true, - "label": "NonProd", - "multi": true, - "name": "NonProd", - "options": [], - "query": "label_values(container_cpu_user_seconds_total{instance=~\"10.115.222.21:8080|10.115.222.22:8080\", name!=\"cadvisor\"}, name)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "definition": "label_values(container_network_transmit_bytes_total{container_label_warnet=\"tank\"}, name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "tanks", - "options": [], - "query": { - "qryType": 5, - "query": "label_values(container_network_transmit_bytes_total{container_label_warnet=\"tank\"}, name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Tank container resource browser", - "uid": "f87c4fd4-7f5b-4ede-b561-b8db42ff095c", - "version": 3, - "weekStart": "" -} diff --git a/src/warnet/templates/grafana-provisioning/dashboards/bitcoin_rpc_scraper.json b/src/warnet/templates/grafana-provisioning/dashboards/bitcoin_rpc_scraper.json deleted file mode 100644 index 4c4dc46e3..000000000 --- a/src/warnet/templates/grafana-provisioning/dashboards/bitcoin_rpc_scraper.json +++ /dev/null @@ -1,340 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "bitcoin_mempool_size", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{job}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Mempool Size (# Txs)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "bitcoin_conn_out", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{job}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Outbound connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheusdatasource" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "bitcoin_conn_in", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{job}}", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Inbound connections", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 38, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Bitcoin RPC scraper", - "uid": "fe65ef87-2bb3-4bef-b57e-0f4d30abdf96", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/src/warnet/templates/grafana-provisioning/dashboards/dashboards.yml b/src/warnet/templates/grafana-provisioning/dashboards/dashboards.yml deleted file mode 100644 index 20cbedccc..000000000 --- a/src/warnet/templates/grafana-provisioning/dashboards/dashboards.yml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 1 -providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /etc/grafana/provisioning/dashboards - diff --git a/src/warnet/templates/grafana-provisioning/datasources/loki.yml b/src/warnet/templates/grafana-provisioning/datasources/loki.yml deleted file mode 100644 index ea5713191..000000000 --- a/src/warnet/templates/grafana-provisioning/datasources/loki.yml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: 5 -datasources: - - name: Loki - type: loki - access: proxy - orgId: 1 - url: http://loki:3100 - basicAuth: false - isDefault: false - version: 1 - editable: false \ No newline at end of file diff --git a/src/warnet/templates/grafana-provisioning/datasources/prometheus.yml b/src/warnet/templates/grafana-provisioning/datasources/prometheus.yml deleted file mode 100644 index 6e5691ee6..000000000 --- a/src/warnet/templates/grafana-provisioning/datasources/prometheus.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: 5 -datasources: - - name: Prometheus - type: prometheus - access: proxy - orgId: 1 - uid: prometheusdatasource - url: http://prometheus:9090 - isDefault: true - diff --git a/src/warnet/templates/rpc/livenessProbe.sh b/src/warnet/templates/rpc/livenessProbe.sh deleted file mode 100755 index 4e5281160..000000000 --- a/src/warnet/templates/rpc/livenessProbe.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -if pgrep -f warnet > /dev/null; then - exit 0 -else - exit 1 -fi diff --git a/src/warnet/utils.py b/src/warnet/utils.py index a02bb39ea..23dad566b 100644 --- a/src/warnet/utils.py +++ b/src/warnet/utils.py @@ -16,8 +16,7 @@ from jsonschema import validate from test_framework.messages import ser_uint256 from test_framework.p2p import MESSAGEMAP - -from .schema import SCHEMA +from warnet import SRC_DIR logger = logging.getLogger("utils") @@ -27,7 +26,6 @@ WEIGHTED_TAGS = [ tag for index, tag in enumerate(reversed(SUPPORTED_TAGS)) for _ in range(index + 1) ] -GRAPH_SCHEMA_PATH = SCHEMA / "graph_schema.json" class NonErrorFilter(logging.Filter): @@ -465,7 +463,7 @@ def convert_unsupported_attributes(graph: nx.Graph): def load_schema(): - with open(GRAPH_SCHEMA_PATH) as schema_file: + with open(SRC_DIR / "graph_schema.json") as schema_file: return json.load(schema_file) diff --git a/src/warnet/warnet.py b/src/warnet/warnet.py index ffadb4ae2..67cd52002 100644 --- a/src/warnet/warnet.py +++ b/src/warnet/warnet.py @@ -5,16 +5,12 @@ import base64 import json import logging -import shutil from pathlib import Path import networkx -import yaml from .backend.kubernetes_backend import KubernetesBackend -from .services import AO_CONF_NAME, FO_CONF_NAME, GRAFANA_PROVISIONING, PROM_CONF_NAME from .tank import Tank -from .templates import TEMPLATES from .utils import gen_config_dir, load_schema, validate_graph_schema logger = logging.getLogger("warnet") @@ -184,85 +180,86 @@ def warnet_down(self): def generate_deployment(self): self.container_interface.generate_deployment_file(self) - if "forkobserver" in self.services: - self.write_fork_observer_config() - if "addrmanobserver" in self.services: - self.write_addrman_observer_config() - if "grafana" in self.services: - self.write_grafana_config() - if "prometheus" in self.services: - self.write_prometheus_config() - def write_fork_observer_config(self): - src = TEMPLATES / FO_CONF_NAME - dst = self.config_dir / FO_CONF_NAME - shutil.copy(src, dst) - with open(dst, "a") as f: - for tank in self.tanks: - f.write( - f""" - [[networks.nodes]] - id = {tank.index} - name = "Node {tank.index}" - description = "Warnet tank {tank.index}" - rpc_host = "{tank.ipv4}" - rpc_port = {tank.rpc_port} - rpc_user = "{tank.rpc_user}" - rpc_password = "{tank.rpc_password}" - """ - ) - logger.info(f"Wrote file: {dst}") + # if "forkobserver" in self.services: + # self.write_fork_observer_config() + # if "addrmanobserver" in self.services: + # self.write_addrman_observer_config() + # if "grafana" in self.services: + # self.write_grafana_config() + # if "prometheus" in self.services: + # self.write_prometheus_config() - def write_addrman_observer_config(self): - src = TEMPLATES / AO_CONF_NAME - dst = self.config_dir / AO_CONF_NAME - shutil.copy(src, dst) - with open(dst, "a") as f: - for tank in self.tanks: - f.write( - f""" - [[nodes]] - id = {tank.index} - name = "node-{tank.index}" - rpc_host = "{tank.ipv4}" - rpc_port = {tank.rpc_port} - rpc_user = "{tank.rpc_user}" - rpc_password = "{tank.rpc_password}" - """ - ) - logger.info(f"Wrote file: {dst}") + # def write_fork_observer_config(self): + # src = FO_CONF_NAME + # dst = self.config_dir / FO_CONF_NAME + # shutil.copy(src, dst) + # with open(dst, "a") as f: + # for tank in self.tanks: + # f.write( + # f""" + # [[networks.nodes]] + # id = {tank.index} + # name = "Node {tank.index}" + # description = "Warnet tank {tank.index}" + # rpc_host = "{tank.ipv4}" + # rpc_port = {tank.rpc_port} + # rpc_user = "{tank.rpc_user}" + # rpc_password = "{tank.rpc_password}" + # """ + # ) + # logger.info(f"Wrote file: {dst}") - def write_grafana_config(self): - src = TEMPLATES / GRAFANA_PROVISIONING - dst = self.config_dir / GRAFANA_PROVISIONING - shutil.copytree(src, dst, dirs_exist_ok=True) - logger.info(f"Wrote directory: {dst}") + # def write_addrman_observer_config(self): + # src = AO_CONF_NAME + # dst = self.config_dir / AO_CONF_NAME + # shutil.copy(src, dst) + # with open(dst, "a") as f: + # for tank in self.tanks: + # f.write( + # f""" + # [[nodes]] + # id = {tank.index} + # name = "node-{tank.index}" + # rpc_host = "{tank.ipv4}" + # rpc_port = {tank.rpc_port} + # rpc_user = "{tank.rpc_user}" + # rpc_password = "{tank.rpc_password}" + # """ + # ) + # logger.info(f"Wrote file: {dst}") - def write_prometheus_config(self): - scrape_configs = [ - { - "job_name": "cadvisor", - "scrape_interval": "15s", - "static_configs": [{"targets": [f"{self.network_name}_cadvisor:8080"]}], - } - ] - for tank in self.tanks: - if tank.exporter: - scrape_configs.append( - { - "job_name": tank.exporter_name, - "scrape_interval": "5s", - "static_configs": [{"targets": [f"{tank.exporter_name}:9332"]}], - } - ) - config = {"global": {"scrape_interval": "15s"}, "scrape_configs": scrape_configs} - prometheus_path = self.config_dir / PROM_CONF_NAME - try: - with open(prometheus_path, "w") as file: - yaml.dump(config, file) - logger.info(f"Wrote file: {prometheus_path}") - except Exception as e: - logger.error(f"An error occurred while writing to {prometheus_path}: {e}") + # def write_grafana_config(self): + # src = GRAFANA_PROVISIONING + # dst = self.config_dir / GRAFANA_PROVISIONING + # shutil.copytree(src, dst, dirs_exist_ok=True) + # logger.info(f"Wrote directory: {dst}") + + # def write_prometheus_config(self): + # scrape_configs = [ + # { + # "job_name": "cadvisor", + # "scrape_interval": "15s", + # "static_configs": [{"targets": [f"{self.network_name}_cadvisor:8080"]}], + # } + # ] + # for tank in self.tanks: + # if tank.exporter: + # scrape_configs.append( + # { + # "job_name": tank.exporter_name, + # "scrape_interval": "5s", + # "static_configs": [{"targets": [f"{tank.exporter_name}:9332"]}], + # } + # ) + # config = {"global": {"scrape_interval": "15s"}, "scrape_configs": scrape_configs} + # prometheus_path = self.config_dir / PROM_CONF_NAME + # try: + # with open(prometheus_path, "w") as file: + # yaml.dump(config, file) + # logger.info(f"Wrote file: {prometheus_path}") + # except Exception as e: + # logger.error(f"An error occurred while writing to {prometheus_path}: {e}") def export(self, config: object, tar_file, exclude: list[int]): for tank in self.tanks: diff --git a/test/dag_connection_test.py b/test/dag_connection_test.py index dba9b8bfb..32c2ccc8c 100755 --- a/test/dag_connection_test.py +++ b/test/dag_connection_test.py @@ -34,7 +34,7 @@ def run_connect_dag_scenario(self): "Successfully ran the connect_dag.py scenario using a temporary file" ] self.log_unexpected_msgs = ["Test failed."] - self.warcli("scenarios run-file test/framework_tests/connect_dag.py") + self.warcli("scenarios run-file test/data/scenario_connect_dag.py") self.wait_for_all_scenarios() self.assert_log_msgs() @@ -42,7 +42,7 @@ def run_connect_dag_scenario_post_connection(self): self.log.info("Running connect_dag scenario") self.log_expected_msgs = ["Successfully ran the connect_dag.py scenario"] self.log_unexpected_msgs = ["Test failed"] - self.warcli("scenarios run-file test/framework_tests/connect_dag.py") + self.warcli("scenarios run-file test/data/scenario_connect_dag.py") self.wait_for_all_scenarios() self.assert_log_msgs() diff --git a/test/framework_tests/connect_dag.py b/test/data/scenario_connect_dag.py similarity index 100% rename from test/framework_tests/connect_dag.py rename to test/data/scenario_connect_dag.py diff --git a/test/test_base.py b/test/test_base.py index 545d07b25..5ad3c9125 100644 --- a/test/test_base.py +++ b/test/test_base.py @@ -1,5 +1,4 @@ import atexit -import importlib.resources as pkg_resources import json import logging import logging.config @@ -11,6 +10,7 @@ from tempfile import mkdtemp from time import sleep +from warnet import SRC_DIR from warnet.cli.rpc import rpc_call from warnet.utils import exponential_backoff from warnet.warnet import Warnet @@ -39,7 +39,7 @@ def setup_environment(self): self.network = True def setup_logging(self): - with pkg_resources.open_text("warnet.logging_config", "config.json") as f: + with open(SRC_DIR / "logging_config.json") as f: logging_config = json.load(f) logging_config["handlers"]["file"]["filename"] = str(self.logfilepath) logging.config.dictConfig(logging_config)