Clean up and organize everything

.github/workflows/deploy.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Build and push production RPC image
         uses: docker/build-push-action@v5
         with:
-          file: src/templates/rpc/Dockerfile_rpc
+          file: resources/images/rpc/Dockerfile_prod
           platforms: linux/amd64,linux/arm64
           context: .
           push: true
@@ -50,7 +50,7 @@ jobs:
         if: github.ref == 'refs/heads/main'
         uses: docker/build-push-action@v5
         with:
-          file: src/templates/rpc/Dockerfile_rpc_dev
+          file: resources/images/rpc/Dockerfile_dev
           platforms: linux/amd64,linux/arm64
           context: .
           push: true

.github/workflows/test.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Build and export
         uses: docker/build-push-action@v5
         with:
-          file: src/warnet/templates/rpc/Dockerfile_rpc
+          file: resources/images/rpc/Dockerfile_prod
           context: .
           tags: warnet/dev
           cache-from: type=gha
@@ -74,13 +74,13 @@ jobs:
           uv pip install -e .
 
           echo "Contents of warnet-rpc-statefulset-dev.yaml being used:"
-          cat src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml
+          cat resources/manifests/warnet-rpc-statefulset-dev.yaml
 
           echo Setting up k8s
-          kubectl apply -f src/warnet/templates/rpc/namespace.yaml
-          kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml
-          kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml
-          kubectl apply -f src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml
+          kubectl apply -f resources/manifests/namespace.yaml
+          kubectl apply -f resources/manifests/rbac-config.yaml
+          kubectl apply -f resources/manifests/warnet-rpc-service.yaml
+          kubectl apply -f resources/manifests/warnet-rpc-statefulset-dev.yaml
           kubectl config set-context --current --namespace=warnet
 
           echo sleeping for 30s to give k8s time to boot

MANIFEST.in

@@ -1,11 +1,2 @@
-include src/warnet/schema/*.json
-include src/warnet/templates/bitcoin.conf
-include src/warnet/templates/fork_observer_config.toml
-include src/warnet/templates/addrman_observer_config.toml
-include src/warnet/templates/addrman.patch
-include src/warnet/templates/isroutable.patch
-include src/warnet/scripts/quick_start.sh
-graft src/warnet/templates/k8s
-graft src/warnet/templates/rpc
-graft src/warnet/templates/grafana-provisioning
-graft src/warnet/logging_config
+include src/warnet/*.json
+graft resources

docs/developer-notes.md

@@ -13,9 +13,9 @@ To help with this a helper script is provided: [build-k8s-rpc.sh](../scripts/bui
 This script can be run in the following way:
 
 ```bash
-DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 ./scripts/build-k8s-rpc.sh Dockerfile_rpc
+DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 ./scripts/build-k8s-rpc.sh Dockerfile_prod
 ```
 
 You can optionally specify `LATEST=1` to also include the `latest` tag on docker hub.
 
-Once a new image has been pushed, it should be referenced in [warnet-rpc-statefulset.yaml](../src/warnet/templates/warnet-rpc-statefulset.yaml) in the `image` field.
+Once a new image has been pushed, it should be referenced in manifests/warnet-rpc-statefulset.yaml in the `image` field.

docs/graph.md

@@ -5,7 +5,7 @@ Warnet creates a Bitcoin network using a network topology from a [graphml](https
 Before any scenarios or RPC commands can be executed, a Warnet network must be started from a graph.
 See [warcli.md](warcli.md) for more details on these commands.
 
-To start a network called `"warnet"` from the [default graph file](../src/graphs/default.graphml):
+To start a network called `"warnet"` from the [default graph file](../graphs/default.graphml):
 ```
 warcli network start
 ```

docs/running.md

@@ -9,44 +9,20 @@ a file `$XDG_STATE_HOME/warnet/warnet.log`, otherwise it will use `$HOME/.warnet
 
 ## Kubernetes
 
-Deploy the resources in `src/warnet/templates/`, this sets up the correct permissions on the cluster (`rbac-config.yaml`) and deploys the warnet RPC server as a service + statefulset.
-
-This can be done with from inside the `src/warnet/templates/` directory by running:
-
-```bash
-kubectl apply -f '*.yaml'
-```
-
-Once the RPC server comes up we need to forward the RPC port from the cluster.
-This can be done with:
-
-```bash
-kubectl port-forward svc/rpc 9276:9276
-```
-
-This allows you to communicate with the RPC server using `warcli`. Developers
-should check the [developer notes](developer-notes.md) to see how to
-update the RPC server when developing on Kubernetes.
-
-Currently, while `warcli network down` will bring down the pods, the RPC server needs manual deletion.
-This can be done using:
-
-```bash
-kubectl delete statefulset
-```
+// TODO
 
 ### Install logging infrastructure
 
 First make sure you have `helm` installed, then simply run the following script:
 
 ```bash
-./src/warnet/templates/k8s/install_logging.sh
+./scripts/install_logging.sh
 ```
 
 To forward port to view Grafana dashboard:
 
 ```bash
-./src/warnet/templates/k8s/connect_logging.sh
+./scripts/connect_logging.sh
 ```
 
 ## Kubernetes (e.g. minikube)

justfile

@@ -5,10 +5,10 @@ default:
     just --list
 
 cluster:
-    kubectl apply -f src/warnet/templates/rpc/namespace.yaml
-    kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml
-    kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml
-    kubectl apply -f src/warnet/templates/rpc/warnet-rpc-statefulset.yaml
+    kubectl apply -f resources/manifests/namespace.yaml
+    kubectl apply -f resources/manifests/rbac-config.yaml
+    kubectl apply -f resources/manifests/warnet-rpc-service.yaml
+    kubectl apply -f resources/manifests/warnet-rpc-statefulset.yaml
 
 # Setup and start the RPC in dev mode with minikube
 start:
@@ -32,14 +32,14 @@ start:
     check_minikube
 
     # Build image in local registry and load into minikube
-    docker build -t warnet/dev -f src/warnet/templates/rpc/Dockerfile_rpc_dev src/warnet/templates/rpc --load
+    docker build -t warnet/dev -f resources/images/rpc/Dockerfile_dev resources/images/rpc --load
     minikube image load warnet/dev
 
     # Setup k8s
-    kubectl apply -f src/warnet/templates/rpc/namespace.yaml
-    kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml
-    kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml
-    kubectl apply -f src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml
+    kubectl apply -f resources/manifests/namespace.yaml
+    kubectl apply -f resources/manifests/rbac-config.yaml
+    kubectl apply -f resources/manifests/warnet-rpc-service.yaml
+    kubectl apply -f resources/manifests/warnet-rpc-statefulset-dev.yaml
     kubectl config set-context --current --namespace=warnet
 
     # Check for warnet-rpc container
@@ -68,11 +68,11 @@ stop:
 
 # Setup and start the RPC in dev mode with Docker Desktop
 startd:
-    docker build -t warnet/dev -f src/warnet/templates/rpc/Dockerfile_rpc_dev src/warnet/templates/rpc --load
-    kubectl apply -f src/warnet/templates/rpc/namespace.yaml
-    kubectl apply -f src/warnet/templates/rpc/rbac-config.yaml
-    kubectl apply -f src/warnet/templates/rpc/warnet-rpc-service.yaml
-    sed 's?/mnt/src?'`PWD`'?g' src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml | kubectl apply -f -
+    docker build -t warnet/dev -f resources/images/rpc/Dockerfile_dev resources/images/rpc --load
+    kubectl apply -f resources/manifests/namespace.yaml
+    kubectl apply -f resources/manifests/rbac-config.yaml
+    kubectl apply -f resources/manifests/warnet-rpc-service.yaml
+    sed 's?/mnt/src?'`PWD`'?g' resouces/manifests/warnet-rpc-statefulset-dev.yaml | kubectl apply -f -
     kubectl config set-context --current --namespace=warnet
 
     echo waiting for rpc to come online
@@ -104,10 +104,10 @@ build branch tag registry=registry repo=repo build-args=build-args action=load:
     warcli image build --registry={{registry}} --repo={{repo}} --branch={{branch}} --arches="{{arches}}" --tag={{tag}} --build-args="{{build-args}}" --action={{action}}
 
 installlogging:
-    ./src/warnet/templates/k8s/install_logging.sh
+    resources/scripts/install_logging.sh
 
 connectlogging:
-    ./src/warnet/templates/k8s/connect_logging.sh
+    resources/scripts/connect_logging.sh
 
 # Format and lint all files
 lint:

pyproject.toml

@@ -32,8 +32,11 @@ build-backend = "setuptools.build_meta"
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}
 
+[tool.setuptools]
+include-package-data = true
+
 [tool.setuptools.packages.find]
-where = ["src"]
+where = ["src", "resources"]
 
 [tool.ruff]
 extend-exclude = [

src/warnet/templates/Dockerfile → resources/images/bitcoin/Dockerfile

@@ -22,8 +22,8 @@ RUN --mount=type=cache,target=/var/cache/apk \
     sqlite-dev \
     zeromq-dev
 
-COPY src/warnet/templates/isroutable.patch /tmp/
-COPY src/warnet/templates/addrman.patch /tmp/
+COPY isroutable.patch /tmp/
+COPY addrman.patch /tmp/
 
 
 # Clone and patch and build stage
@@ -77,8 +77,7 @@ RUN --mount=type=cache,target=/var/cache/apk sed -i 's/http\:\/\/dl-cdn.alpineli
     su-exec
 
 COPY --from=build /opt/bitcoin /usr/local
-COPY src/warnet/templates/entrypoint.sh /entrypoint.sh
-COPY src/warnet/templates/tor/torrc /etc/tor/warnet-torr
+COPY entrypoint.sh /
 
 VOLUME ["/home/bitcoin/.bitcoin"]
 EXPOSE 8332 8333 18332 18333 18443 18444 38333 38332

src/warnet/templates/rpc/Dockerfile_rpc_dev → resources/images/rpc/Dockerfile_dev

@@ -20,13 +20,10 @@ EXPOSE 9276
 # we defer this to the entrypoint script for dev mode to enable hot-reloading.
 
 # Copy the entrypoint script into the container
-COPY entrypoint.sh /usr/local/bin/
-
-# Make the entrypoint script executable
-RUN chmod +x /usr/local/bin/entrypoint.sh
+COPY entrypoint.sh /
 
 # Set the entrypoint script to run when the container launches
-ENTRYPOINT ["entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]
 
 # Default command
 CMD ["warnet", "--dev"]

src/warnet/templates/rpc/warnet-rpc-statefulset-dev.yaml → resources/manifests/warnet-rpc-statefulset-dev.yaml

@@ -26,7 +26,15 @@ spec:
         livenessProbe:
           # fail (restart) the pod if we can't find pid of warnet
           exec:
-            command: [ "/bin/bash", "-c", "/root/warnet/src/warnet/templates/rpc/livenessProbe.sh" ]
+            command:
+              - /bin/sh
+              - -c
+              - |
+                if pgrep -f warnet > /dev/null; then
+                    exit 0
+                else
+                    exit 1
+                fi
           initialDelaySeconds: 20
           periodSeconds: 5
           failureThreshold: 3

scripts/build-k8s-rpc.sh → resources/scripts/build-k8s-rpc.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Run with e.g.:
-# $ DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 LATEST=1 ./scripts/build-k8s-rpc.sh Dockerfile_rpc
+# $ DOCKER_REGISTRY=bitcoindevproject/warnet-rpc TAG=0.1 LATEST=1 ./scripts/build-k8s-rpc.sh Dockerfile_prod
 
 # Fail on any step
 set -ex

src/warnet/templates/k8s/install_logging.sh → resources/scripts/install_logging.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 set -e
 
-SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+MANIFESTS_DIR=$( cd -- "$THIS_DIR/../manifests" &> /dev/null && pwd )
 
 helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm repo update
 
-helm upgrade --install --namespace warnet-logging --create-namespace --values "${SCRIPT_DIR}/loki/values.yaml" loki grafana/loki --version 5.47.2
+helm upgrade --install --namespace warnet-logging --create-namespace --values "${MANIFESTS_DIR}/loki_values.yaml" loki grafana/loki --version 5.47.2
 helm upgrade --install --namespace warnet-logging promtail grafana/promtail
 helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false
-helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${SCRIPT_DIR}/grafana/values.yaml"
+helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values "${MANIFESTS_DIR}/grafana_values.yaml"

src/warnet/__init__.py

@@ -0,0 +1,3 @@
+from importlib.resources import files
+
+SRC_DIR = files("warnet")