Skip to content

Commit a114d89

Browse files
committed
circuit breaker setup
1 parent 4f4bde8 commit a114d89

File tree

7 files changed

+181
-92
lines changed

7 files changed

+181
-92
lines changed

resources/networks/hello/network.yaml

Lines changed: 4 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,6 @@ plugins: # Each plugin section has a number of hooks available (preDeploy, post
5959
entrypoint: "../../plugins/hello" # This entrypoint path is relative to the network.yaml file
6060
podName: "hello-pre-deploy"
6161
helloTo: "preDeploy!"
62-
circuitbreaker:
63-
entrypoint: "../../plugins/circuitbreaker"
64-
url: "http://127.0.0.1:9235"
65-
apiUrl: "/api"
66-
mode: "fail" # Operating mode: fail, queue, or queue_peer_initiated
67-
maxPendingHtlcs: 10 # Default maximum pending HTLCs per peer
68-
rateLimit: 1 # Minimum seconds between HTLCs (token bucket rate limit)
6962
postDeploy:
7063
hello:
7164
entrypoint: "../../plugins/hello"
@@ -76,54 +69,24 @@ plugins: # Each plugin section has a number of hooks available (preDeploy, post
7669
activity: '[{"source": "tank-0003-ln", "destination": "tank-0005-ln", "interval_secs": 1, "amount_msat": 2000}]'
7770
circuitbreaker:
7871
entrypoint: "../../plugins/circuitbreaker"
79-
url: "http://127.0.0.1:9235"
80-
apiUrl: "/api"
81-
mode: "fail" # Operating mode: fail, queue, or queue_peer_initiated
82-
maxPendingHtlcs: 10 # Default maximum pending HTLCs per peer
83-
rateLimit: 1 # Minimum seconds between HTLCs (token bucket rate limit)
72+
podName: "circuitbreaker-pod"
73+
rpcserver: "tank-0000-ln:10009"
74+
httplisten: "0.0.0.0:9235"
8475
preNode: # preNode plugins run before each node is deployed
8576
hello:
8677
entrypoint: "../../plugins/hello"
8778
helloTo: "preNode!"
88-
circuitbreaker:
89-
entrypoint: "../../plugins/circuitbreaker"
90-
url: "http://127.0.0.1:9235"
91-
apiUrl: "/api"
92-
mode: "fail" # Operating mode: fail, queue, or queue_peer_initiated
93-
maxPendingHtlcs: 10 # Default maximum pending HTLCs per peer
94-
rateLimit: 1 # Minimum seconds between HTLCs (token bucket rate limit)
9579
postNode:
9680
hello:
9781
entrypoint: "../../plugins/hello"
9882
helloTo: "postNode!"
99-
circuitbreaker:
100-
entrypoint: "../../plugins/circuitbreaker"
101-
url: "http://127.0.0.1:9235"
102-
apiUrl: "/api"
103-
mode: "fail" # Operating mode: fail, queue, or queue_peer_initiated
104-
maxPendingHtlcs: 10 # Default maximum pending HTLCs per peer
105-
rateLimit: 1 # Minimum seconds between HTLCs (token bucket rate limit)
10683
preNetwork:
10784
hello:
10885
entrypoint: "../../plugins/hello"
10986
helloTo: "preNetwork!"
11087
podName: "hello-pre-network"
111-
circuitbreaker:
112-
entrypoint: "../../plugins/circuitbreaker"
113-
url: "http://127.0.0.1:9235"
114-
apiUrl: "/api"
115-
mode: "fail" # Operating mode: fail, queue, or queue_peer_initiated
116-
maxPendingHtlcs: 10 # Default maximum pending HTLCs per peer
117-
rateLimit: 1 # Minimum seconds between HTLCs (token bucket rate limit)
11888
postNetwork:
11989
hello:
12090
entrypoint: "../../plugins/hello"
12191
helloTo: "postNetwork!"
122-
podName: "hello-post-network"
123-
circuitbreaker:
124-
entrypoint: "../../plugins/circuitbreaker"
125-
url: "http://127.0.0.1:9235"
126-
apiUrl: "/api"
127-
mode: "fail" # Operating mode: fail, queue, or queue_peer_initiated
128-
maxPendingHtlcs: 10 # Default maximum pending HTLCs per peer
129-
rateLimit: 1 # Minimum seconds between HTLCs (token bucket rate limit)
92+
podName: "hello-post-network"

resources/plugins/circuitbreaker/charts/circuitbreaker/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@ apiVersion: v2
22
name: circuitbreaker
33
description: A Helm chart to deploy Circuit Breaker
44
version: 0.1.0
5-
appVersion: "0.1.0"
5+
appVersion: "0.1.0"
Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ .Values.podName }}
5+
spec:
6+
replicas: 1
7+
selector:
8+
matchLabels:
9+
app: circuitbreaker
10+
template:
11+
metadata:
12+
labels:
13+
app: circuitbreaker
14+
spec:
15+
containers:
16+
- name: circuitbreaker
17+
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
18+
args:
19+
- "--rpcserver={{ .Values.rpcserver }}"
20+
- "--httplisten={{ .Values.httplisten }}"
21+
ports:
22+
- containerPort: 9235
23+
volumeMounts:
24+
- name: lnd-tls-cert
25+
mountPath: /root/.lnd/tls.cert
26+
subPath: tls.cert
27+
- name: lnd-macaroon
28+
mountPath: /root/.lnd/data/chain/bitcoin/regtest/admin.macaroon
29+
subPath: admin.macaroon
30+
volumes:
31+
- name: lnd-tls-cert
32+
secret:
33+
secretName: lnd-tls-cert-{{ .Values.podName }}
34+
- name: lnd-macaroon
35+
secret:
36+
secretName: lnd-macaroon-{{ .Values.podName }}

resources/plugins/circuitbreaker/charts/circuitbreaker/templates/pod.yaml

Lines changed: 0 additions & 16 deletions
This file was deleted.
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: {{ .Values.podName }}
5+
spec:
6+
selector:
7+
app: circuitbreaker
8+
ports:
9+
- protocol: TCP
10+
port: 9235
11+
targetPort: 9235

resources/plugins/circuitbreaker/charts/circuitbreaker/values.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
name: "circuitbreaker"
1+
podName: "circuitbreaker-pod"
2+
rpcserver: "localhost:10009"
3+
httplisten: "0.0.0.0:9235"
24
image:
35
repository: "camillarhi/circuitbreaker"
46
tag: "latest"

resources/plugins/circuitbreaker/plugin.py

Lines changed: 126 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import logging
44
from enum import Enum
55
from pathlib import Path
6+
import subprocess
67
import time
78
from typing import Optional
89

@@ -25,11 +26,9 @@
2526

2627
PLUGIN_DIR_TAG = "plugin_dir"
2728

28-
2929
class PluginError(Exception):
3030
pass
3131

32-
3332
log = logging.getLogger(MISSION)
3433
if not log.hasHandlers():
3534
console_handler = logging.StreamHandler()
@@ -41,9 +40,9 @@ class PluginError(Exception):
4140
log.propagate = True
4241

4342
class PluginContent(Enum):
44-
MODE = "mode"
45-
MAX_PENDING_HTLCS = "maxPendingHtlcs"
46-
RATE_LIMIT = "rateLimit"
43+
POD_NAME = "podName"
44+
LND_RPC_SERVER = "rpcserver"
45+
HTTP_LISTEN = "httplisten"
4746

4847
@click.group()
4948
@click.pass_context
@@ -85,47 +84,141 @@ def _entrypoint(ctx, plugin_content: dict, warnet_content: dict):
8584
hook_value = warnet_content[WarnetContent.HOOK_VALUE.value]
8685

8786
match hook_value:
88-
case (
89-
HookValue.PRE_NETWORK
90-
| HookValue.POST_NETWORK
91-
| HookValue.PRE_DEPLOY
92-
| HookValue.POST_DEPLOY
93-
):
94-
data = get_data(plugin_content)
95-
if data:
96-
_launch_circuit_breaker(ctx, node_name=hook_value.value.lower()+"breaker",hook_value=hook_value.value)
97-
else:
98-
_launch_circuit_breaker(ctx, node_name=hook_value.value.lower()+"breaker",hook_value=hook_value.value)
99-
case HookValue.PRE_NODE:
100-
name = warnet_content[PLUGIN_ANNEX][AnnexMember.NODE_NAME.value] + "-pre-pod"
101-
_launch_circuit_breaker(ctx, node_name=hook_value.value.lower() + "-" + name, hook_value=hook_value.value)
102-
case HookValue.POST_NODE:
103-
name = warnet_content[PLUGIN_ANNEX][AnnexMember.NODE_NAME.value] + "-post-pod"
104-
_launch_circuit_breaker(ctx, node_name=hook_value.value.lower() + "-" + name, hook_value=hook_value.value)
87+
case HookValue.POST_DEPLOY:
88+
# data = get_data(plugin_content)
89+
# if data:
90+
# log.info(f"Launching circuit breaker with data: {data}")
91+
# _create_secrets()
92+
_launch_circuit_breaker(ctx, plugin_content)
93+
# else:
94+
# _launch_circuit_breaker(ctx, install_name="circuitbreaker")
95+
case _:
96+
log.info(f"No action required for hook {hook_value}")
10597

10698
def get_data(plugin_content: dict) -> Optional[dict]:
10799
data = {
108100
key: plugin_content.get(key)
109-
for key in (PluginContent.MAX_PENDING_HTLCS.value, PluginContent.RATE_LIMIT.value)
101+
for key in (PluginContent.POD_NAME.value, PluginContent.LND_RPC_SERVER.value, PluginContent.HTTP_LISTEN.value)
110102
if plugin_content.get(key)
111103
}
112104
return data or None
113105

106+
# def _create_secrets():
107+
# """Use local LND files for testing"""
108+
# log.info("Using local LND files for testing")
109+
# tls_cert_path = Path.home() / ".lnd" / "tls.cert"
110+
# admin_macaroon_path = Path.home() / ".lnd" / "data" / "chain" / "bitcoin" / "signet" / "admin.macaroon"
114111

115-
def _launch_circuit_breaker(ctx, node_name: str, hook_value: str):
112+
# if not tls_cert_path.exists():
113+
# raise PluginError(f"TLS certificate not found at {tls_cert_path}")
114+
# if not admin_macaroon_path.exists():
115+
# raise PluginError(f"Admin macaroon not found at {admin_macaroon_path}")
116+
117+
# log.info(f"Using TLS certificate: {tls_cert_path}")
118+
# log.info(f"Using admin macaroon: {admin_macaroon_path}")
119+
120+
# def _create_secrets():
121+
# """Create Kubernetes secrets for each LND node"""
122+
# lnd_pods = subprocess.check_output(["kubectl", "get", "pods", "-l", "mission=lightning", "-o", "name"]).decode().splitlines()
123+
# # lnd_pods = subprocess.check_output(["kubectl", "get", "pods", "-l", "app=warnet", "-l", "mission=lightning", "-o", "name"]).decode().splitlines()
124+
# for node in lnd_pods:
125+
# node_name = node.split('/')[-1]
126+
# log.info(f"Waiting for {node_name} to be ready...")
127+
# wait_for_init(node_name, namespace=get_default_namespace(), quiet=True)
128+
# log.info(f"Creating secrets for {node_name}")
129+
# subprocess.run(["kubectl", "cp", f"{node}:/root/.lnd/tls.cert", "./tls.cert"], check=True)
130+
# subprocess.run(["kubectl", "cp", f"{node}:/root/.lnd/data/chain/bitcoin/regtest/admin.macaroon", "./admin.macaroon"], check=True)
131+
# subprocess.run(["kubectl", "create", "secret", "generic", f"lnd-tls-cert-{node_name}", "--from-file=tls.cert=./tls.cert"], check=True)
132+
# subprocess.run(["kubectl", "create", "secret", "generic", f"lnd-macaroon-{node_name}", "--from-file=admin.macaroon=./admin.macaroon"], check=True)
133+
134+
def _create_secrets():
135+
"""Create Kubernetes secrets for each LND node"""
136+
lnd_pods = subprocess.check_output(
137+
["kubectl", "get", "pods", "-l", "mission=lightning", "-o", "name"]
138+
).decode().splitlines()
139+
140+
for node in lnd_pods:
141+
node_name = node.split('/')[-1]
142+
log.info(f"Waiting for {node_name} to be ready...")
143+
144+
# Wait for the pod to be ready
145+
max_retries = 10
146+
retry_delay = 10 # seconds
147+
for attempt in range(max_retries):
148+
try:
149+
# Check if the pod is ready
150+
pod_status = subprocess.check_output(
151+
["kubectl", "get", "pod", node_name, "-o", "jsonpath='{.status.phase}'"]
152+
).decode().strip("'")
153+
154+
if pod_status == "Running":
155+
log.info(f"{node_name} is ready.")
156+
break
157+
else:
158+
log.info(f"{node_name} is not ready yet (status: {pod_status}). Retrying in {retry_delay} seconds...")
159+
except subprocess.CalledProcessError as e:
160+
log.error(f"Failed to check pod status for {node_name}: {e}")
161+
if attempt == max_retries - 1:
162+
raise PluginError(f"Pod {node_name} did not become ready after {max_retries} attempts.")
163+
164+
time.sleep(retry_delay)
165+
166+
# Create secrets for the pod
167+
log.info(f"Creating secrets for {node_name}")
168+
try:
169+
subprocess.run(
170+
["kubectl", "cp", f"{node_name}:/root/.lnd/tls.cert", "./tls.cert"],
171+
check=True
172+
)
173+
subprocess.run(
174+
["kubectl", "cp", f"{node_name}:/root/.lnd/data/chain/bitcoin/regtest/admin.macaroon", "./admin.macaroon"],
175+
check=True
176+
)
177+
subprocess.run(
178+
["kubectl", "create", "secret", "generic", f"lnd-tls-cert-{node_name}", "--from-file=tls.cert=./tls.cert"],
179+
check=True
180+
)
181+
subprocess.run(
182+
["kubectl", "create", "secret", "generic", f"lnd-macaroon-{node_name}", "--from-file=admin.macaroon=./admin.macaroon"],
183+
check=True
184+
)
185+
except subprocess.CalledProcessError as e:
186+
log.error(f"Failed to create secrets for {node_name}: {e}")
187+
raise PluginError(f"Failed to create secrets for {node_name}.")
188+
189+
def _launch_circuit_breaker(ctx,
190+
plugin_content: dict,
191+
install_name: str="circuitbreaker",
192+
podName: str ="circuitbreaker-pod",
193+
rpcserver: str = "localhost:10009",
194+
httplisten: str = "0.0.0.0:9235"):
116195
timestamp = int(time.time())
117-
release_name = f"cb-{node_name}"
196+
# release_name = f"cb-{install_name}"
118197

198+
lnd_pods = subprocess.check_output(["kubectl", "get", "pods", "-l", "app=warnet", "-l", "mission=lightning", "-o", "name"]).decode().splitlines()
199+
for node in lnd_pods:
200+
node_name = node.split('/')[-1]
201+
log.info(f"Launching Circuit Breaker for {node_name}")
202+
release_name = f"circuitbreaker-{node_name}"
203+
204+
command = (
205+
f"helm upgrade --install {release_name} {ctx.obj[PLUGIN_DIR_TAG]}/charts/circuitbreaker "
206+
f"--set podName={release_name} --set rpcserver=localhost:10009 --set httplisten=0.0.0.0:9235"
207+
)
208+
119209
# command = f"helm upgrade --install {release_name} {ctx.obj[PLUGIN_DIR_TAG]}/charts/circuitbreaker"
120-
command = (
121-
f"helm upgrade --install {release_name} {ctx.obj[PLUGIN_DIR_TAG]}/charts/circuitbreaker "
122-
f"--set name={release_name}"
123-
)
124-
log.info(command)
125-
run_command(command)
210+
# command = (
211+
# f"helm upgrade --install {install_name} {ctx.obj[PLUGIN_DIR_TAG]}/charts/circuitbreaker "
212+
# f"--set podName={podName} --set rpcserver={rpcserver} --set httplisten={httplisten}"
213+
# )
214+
log.info(command)
215+
try:
216+
run_command(command)
126217

127-
if(hook_value==HookValue.POST_DEPLOY):
128-
wait_for_init(release_name, namespace=get_default_namespace(), quiet=True)
218+
# if(hook_value==HookValue.POST_DEPLOY):
219+
wait_for_init(release_name, namespace=get_default_namespace(), quiet=True)
220+
except Exception as e:
221+
log.error(f"Failed to launch Circuit Breaker for {node_name}: {e}")
129222

130223

131224
if __name__ == "__main__":

0 commit comments

Comments
 (0)