Merge pull request #419 from willcl-ark/package-fixups

`warcli cluster` refactors

Author: m3dwards

docs/quickrun.md

@@ -1,44 +1,66 @@
 # Quick run
 
-Warnet runs a server which can be used to manage multiple networks. On docker
-this runs locally, but on Kubernetes this runs as a `statefulSet` in the
-cluster.
+## Installation
 
-If the `$XDG_STATE_HOME` environment variable is set, the server will log to
-a file `$XDG_STATE_HOME/warnet/warnet.log`, otherwise it will use `$HOME/.warnet/warnet.log`.
+Either install warnet via pip, or clone the source and install:
 
-## Quick start via pip
+### via pip
 
-You can install warnet via `pip` into your virtual environment with
+You can install warnet via `pip` into a virtual environment with
 
 ```bash
 python3 -m venv .venv
 source .venv/bin/activate
 pip install warnet
 ```
 
-Following installation `warcli` commands will operate natively on the Kubernetes cluster currently configured with `kubectl`.
+### via cloned source
 
-Starting the Warnet server is as easy as:
+You can install warnet from source into a virtual environment with
 
 ```bash
-# (optional) if using a local minikube cluster check that we have all required programs installed
-warcli setup
+git clone https://github.com/bitcoin-dev-project/warnet.git
+cd warnet
+python3 -m venv .venv
+source .venv/bin/activate
+pip install -e .
+```
+
+## Running
+
+> [!TIP]
+> When developing locally add the `--dev` flag to `warcli cluster deploy` to enable dev mode with hot-reloading server.
+
+### Using minikube
+
+To run a local cluster using minikube:
+
+```bash
+warcli cluster setup-minikube
 
-# (optional) if using a local minikube cluster, set it up
-warcli cluster minikube-setup
+warcli cluster deploy
+```
+
+### Other cluster types
 
+If not using minikube (e.g. using Docker Desktop or a managed cluster), `warcli` commands will operate natively on the current Kubernetes context, so you can simply run:
+
+```bash
 warcli cluster deploy
 ```
 
-This also automatically configures port forwarding to the Server in the cluster.
+...to deploy warnet to your cluster.
+
+`warcli deploy` also automatically configures port forwarding to the Server in the cluster.
+
+## Stopping
 
 To tear down the cluster:
 
 ```bash
 warcli cluster teardown
+```
 
-# (optional) if using a local minikube cluster, remove the image
-warcli cluster minikube-clean
-
+## Log location
 
+If the `$XDG_STATE_HOME` environment variable is set, the server will log to a file `$XDG_STATE_HOME/warnet/warnet.log`, otherwise it will use `$HOME/.warnet/warnet.log`.

docs/warcli.md

@@ -26,6 +26,10 @@ options:
 |----------|--------|------------|-----------|
 | commands | String |            |           |
 
+### `warcli setup`
+Check Warnet requirements are installed
+
+
 ## Bitcoin
 
 ### `warcli bitcoin debug-log`
@@ -69,16 +73,20 @@ options:
 
 ## Cluster
 
-### `warcli cluster deploy`
-Setup Warnet using the current kubectl-configured cluster
+### `warcli cluster connect-logging`
+Connect kubectl to cluster logging
 
 
-### `warcli cluster minikube-clean`
-Reinit minikube images
+### `warcli cluster deploy`
+Deploy Warnet using the current kubectl-configured cluster
 
+options:
+| name   | type   | required   | default   |
+|--------|--------|------------|-----------|
+| dev    | Bool   |            | False     |
 
-### `warcli cluster minikube-setup`
-Setup minikube for use with Warnet
+### `warcli cluster deploy-logging`
+Deploy logging configurations to the cluster using helm
 
 
 ### `warcli cluster port-start`
@@ -89,6 +97,14 @@ Port forward (runs as a detached process)
 Stop the port forwarding process
 
 
+### `warcli cluster setup-minikube`
+Configure a local minikube cluster
+
+options:
+| name   | type   | required   | default   |
+|--------|--------|------------|-----------|
+| clean  | Bool   |            | False     |
+
 ### `warcli cluster teardown`
 Stop the warnet server and tear down the cluster
 

justfile

@@ -1,98 +1,30 @@
 set shell := ["bash", "-uc"]
+default_cpus := "4"
+default_memory := "4000"
 
 [private]
 default:
     just --list
 
-cluster:
-    kubectl apply -f resources/manifests/namespace.yaml
-    kubectl apply -f resources/manifests/rbac-config.yaml
-    kubectl apply -f resources/manifests/warnet-rpc-service.yaml
-    kubectl apply -f resources/manifests/warnet-rpc-statefulset.yaml
+# Configure a minikube k8s cluster
+minikube cpus=default_cpus memory=default_memory:
+    WAR_RPC="./resources/images/rpc" WAR_CPU={{ cpus }} WAR_MEM={{ memory }} ./resources/scripts/setup_minikube.sh
 
-# Setup and start the RPC in dev mode with minikube
-start:
-    #!/usr/bin/env bash
-    set -euxo pipefail
+# Deploy Warnet in dev mode
+deploy:
+    WAR_MANIFESTS="./resources/manifests" WAR_DEV=1 ./resources/scripts/deploy.sh
 
-    # Function to check if minikube is running
-    check_minikube() {
-        minikube status | grep -q "Running" && echo "Minikube is already running" || minikube start --mount --mount-string="$PWD:/mnt/src"
-    }
-
-    # Function to check if warnet-rpc container is already running
-    check_warnet_rpc() {
-        if kubectl get pods --all-namespaces | grep -q "bitcoindevproject/warnet-rpc"; then
-            echo "warnet-rpc already running in minikube"
-            exit 1
-        fi
-    }
-
-    # Check minikube status
-    check_minikube
-
-    # Build image in local registry and load into minikube
-    docker build -t warnet/dev -f resources/images/rpc/Dockerfile_dev resources/images/rpc --load
-    minikube image load warnet/dev
-
-    # Setup k8s
-    kubectl apply -f resources/manifests/namespace.yaml
-    kubectl apply -f resources/manifests/rbac-config.yaml
-    kubectl apply -f resources/manifests/warnet-rpc-service.yaml
-    kubectl apply -f resources/manifests/warnet-rpc-statefulset-dev.yaml
-    kubectl config set-context --current --namespace=warnet
-
-    # Check for warnet-rpc container
-    check_warnet_rpc
-
-    until kubectl get pod rpc-0 --namespace=warnet; do
-       echo "Waiting for server to find pod rpc-0..."
-       sleep 4
-    done
-
-    echo "⏲️ This could take a minute or so."
-    kubectl wait --for=condition=Ready --timeout=2m pod rpc-0
-
-    echo Done...
-
-# Stop the RPC in dev mode with minikube
-stop:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-
-    kubectl delete namespace warnet
-    kubectl delete namespace warnet-logging --ignore-not-found
-    kubectl config set-context --current --namespace=default
-
-    minikube image rm warnet/dev
-
-# Setup and start the RPC in dev mode with Docker Desktop
-startd:
-    docker build -t warnet/dev -f resources/images/rpc/Dockerfile_dev resources/images/rpc --load
-    kubectl apply -f resources/manifests/namespace.yaml
-    kubectl apply -f resources/manifests/rbac-config.yaml
-    kubectl apply -f resources/manifests/warnet-rpc-service.yaml
-    sed 's?/mnt/src?'`PWD`'?g' resources/manifests/warnet-rpc-statefulset-dev.yaml | kubectl apply -f -
-    kubectl config set-context --current --namespace=warnet
-
-    echo waiting for rpc to come online
-    kubectl wait --for=condition=Ready --timeout=2m pod rpc-0
-
-    echo Done...
-
-# Stop the RPC in dev mode with Docker Desktop
-stopd:
-    # Delete all resources
-    kubectl delete namespace warnet
-    kubectl delete namespace warnet-logging --ignore-not-found
-    kubectl config set-context --current --namespace=default
-
-    echo Done...
+# Stop and teardown warnet
+teardown:
+    ./resources/scripts/stop.sh
 
 # port forward
 p:
     kubectl port-forward svc/rpc 9276:9276
 
+# Quick start for minikube local dev
+qs: minikube deploy p
+
 registry := 'bitcoindevproject/bitcoin'
 repo := 'bitcoin/bitcoin'
 arches := 'amd64,arm64'
@@ -104,10 +36,10 @@ build branch tag registry=registry repo=repo build-args=build-args action=load:
     warcli image build --registry={{registry}} --repo={{repo}} --branch={{branch}} --arches="{{arches}}" --tag={{tag}} --build-args="{{build-args}}" --action={{action}}
 
 installlogging:
-    resources/scripts/install_logging.sh
+    ./resources/scripts/install_logging.sh
 
 connectlogging:
-    resources/scripts/connect_logging.sh
+    ./resources/scripts/connect_logging.sh
 
 # Format and lint all files
 lint:

resources/scripts/deploy.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+
+if [ -z "${WAR_MANIFESTS+x}" ]; then
+    echo "WAR_MANIFESTS is unset. Please provide a path to warnet manifests."
+    exit 1
+fi
+
+# Function to check if warnet-rpc container is already running
+check_warnet_rpc() {
+    if kubectl get pods --all-namespaces | grep -q "bitcoindevproject/warnet-rpc"; then
+        echo "warnet-rpc pod found"
+        exit 1
+    fi
+}
+
+# Deploy base configurations
+kubectl apply -f "$WAR_MANIFESTS/namespace.yaml"
+kubectl apply -f "$WAR_MANIFESTS/rbac-config.yaml"
+kubectl apply -f "$WAR_MANIFESTS/warnet-rpc-service.yaml"
+
+# Deploy rpc server
+if [ -n "${WAR_DEV+x}" ]; then # Dev mode selector
+    # Build image in local registry
+    docker build -t warnet/dev -f "$WAR_RPC/Dockerfile_dev" "$WAR_RPC" --load
+    if [ "$(kubectl config current-context)" = "docker-desktop" ]; then
+        sed "s?/mnt/src?$(pwd)?g" "$WAR_MANIFESTS/warnet-rpc-statefulset-dev.yaml" | kubectl apply -f -
+    else # assuming minikube
+        minikube image load warnet/dev
+        kubectl apply -f "$WAR_MANIFESTS/warnet-rpc-statefulset-dev.yaml"
+    fi
+else
+    kubectl apply -f "$WAR_MANIFESTS/warnet-rpc-statefulset.yaml"
+fi
+
+kubectl config set-context --current --namespace=warnet
+
+# Check for warnet-rpc container
+check_warnet_rpc
+
+until kubectl get pod rpc-0 --namespace=warnet; do
+   echo "Waiting for server to find pod rpc-0..."
+   sleep 4
+done
+
+echo "⏲️ This could take a minute or so."
+kubectl wait --for=condition=Ready --timeout=2m pod rpc-0
+
+echo Done...

resources/scripts/quick_start.sh

@@ -38,7 +38,7 @@ print_partial_message() {
     local pre_message="$1"
     local formatted_part="$2"
     local post_message="$3"
-    local format="$4"
+    local format="${4:-}"  # Default to empty string if not provided
     local color="${5:-$RESET}"
 
     echo -e "${color}${pre_message}${format}${formatted_part}${RESET}${color}${post_message}${RESET}"
@@ -103,7 +103,6 @@ if [ -n "$just_path" ]; then
 else
     print_partial_message " 💥 Could not find " "just" ". Please follow this link to install it..." "$BOLD"
     print_message "" "   https://github.com/casey/just?tab=readme-ov-file#pre-built-binaries" "$BOLD"
-    exit 127
 fi
 
 python_path=$(command -v python3 || true)
@@ -115,11 +114,10 @@ else
     exit 127
 fi
 
-venv_status=$(python3 -m venv --help || true)
-if [ -n "$venv_status" ]; then
-    print_partial_message " ⭐️ Found " "venv" ": a python3 module" "$BOLD"
+if [ -n "$VIRTUAL_ENV" ]; then
+    print_partial_message " ⭐️ Running in virtual environment: " "$VIRTUAL_ENV" "$BOLD"
 else
-    print_partial_message " 💥 Could not find " "venv" ". Please install it using your package manager." "$BOLD"
+    print_partial_message " 💥 Not running in a virtual environment. " "Please activate a venv before proceeding." "$BOLD"
     exit 127
 fi
 
@@ -136,44 +134,3 @@ else
     print_partial_message " 💥 Could not find " "BPF" ". Please figure out how to enable Berkeley Packet Filters in your kernel." "$BOLD"
     exit 1
 fi
-
-print_message "" "" ""
-print_message "" "    Let's try to spin up a python virtual environment..." ""
-print_message "" "" ""
-
-if [ -d ".venv" ]; then
-    print_message "" "    It looks like a virtual environment already exists!" ""
-else
-    print_message "" "    Creating a new virtual environment..." ""
-    python3 -m venv .venv
-fi
-
-source .venv/bin/activate
-
-print_message "" "" ""
-print_partial_message " ⭐️ " "venv" ": The python virtual environment looks good" "$BOLD"
-print_message "" "" ""
-print_message "" "    Let's install warnet into that virtual environment..." ""
-print_message "" "" ""
-
-pip install --upgrade pip
-pip install -e .
-
-print_message "" "" ""
-print_partial_message " ⭐️ " "warnet" ": We installed Warnet in the virtual environment" "$BOLD"
-print_message "" "" ""
-print_message "" "    Now, let's get the Warnet started..." ""
-print_message "" "" ""
-
-just start
-just p &
-sleep 1
-warcli network start
-sleep 1
-while warcli network connected | grep -q "False"; do
-    sleep 2
-done
-print_message "" "🥳" ""
-print_message "" "Run the following command to enter into the python virtual environment..." ""
-print_message "" "    source .venv/bin/activate" "$BOLD"
-print_partial_message "   After that, you can run " "warcli help" " to start running Warnet commands." "$BOLD"