Merge pull request #691 from macgyver13/lnd-exporter

Enable lnd pod logging

Author: pinheadmz

docs/logging_monitoring.md

@@ -116,6 +116,16 @@ outbounds 0.0
 mempool_size 0.0
 ```
 
+### Defining lnd metrics to capture
+
+Lightning nodes can also be configured to export metrics to prometheus using `lnd-exporter`.
+Example configuration is provided in `test/data/ln/`. Review `node-defauts.yaml` for a typical logging configuration. All default metrics reported to prometheus are prefixed with `lnd_`
+
+[lnd-exporter configuration reference](https://github.com/macgyver13/lnd-exporter/tree/main?tab=readme-ov-file#configuration)
+lnd-exporter assumes same macaroon referenced in ln_framework (can be overridden by env variable)
+
+**Note: `test/data/ln` and `test/data/logging` take advantage of **extraContainers** configuration option to add containers to default `lnd/templates/pod`*
+
 ### Grafana
 
 Data from Prometheus exporters is collected and fed into Grafana for a

resources/charts/bitcoincore/charts/lnd/templates/configmap.yaml

@@ -4,6 +4,7 @@ metadata:
   name: {{ include "lnd.fullname" . }}
   labels:
     {{- include "lnd.labels" . | nindent 4 }}
+    role: macaroon-ref
 data:
   lnd.conf: |
     {{- .Values.baseConfig | nindent 4 }}
@@ -37,7 +38,7 @@ data:
     AwEHoUQDQgAEBVltIvaTlAQI/3FFatTqVflZuZdRJ0SmRMSJrFLPtp0fxE7hmteS
     t6gjQriy90fP8j9OJXBNAjt915kLY4zVvg==
     -----END EC PRIVATE KEY-----
-
+  MACAROON_HEX: 0201036c6e6402f801030a1062beabbf2a614b112128afa0c0b4fdd61201301a160a0761646472657373120472656164120577726974651a130a04696e666f120472656164120577726974651a170a08696e766f69636573120472656164120577726974651a210a086d616361726f6f6e120867656e6572617465120472656164120577726974651a160a076d657373616765120472656164120577726974651a170a086f6666636861696e120472656164120577726974651a160a076f6e636861696e120472656164120577726974651a140a057065657273120472656164120577726974651a180a067369676e6572120867656e657261746512047265616400000620b17be53e367290871681055d0de15587f6d1cd47d1248fe2662ae27f62cfbdc6
 ---
 apiVersion: v1
 kind: ConfigMap

resources/charts/bitcoincore/charts/lnd/templates/pod.yaml

@@ -12,6 +12,8 @@ metadata:
     collect_logs: "true"
     {{- end }}
     chain: {{ .Values.global.chain }}
+  annotations:
+    kubectl.kubernetes.io/default-container: "lnd"
 spec:
   {{- with .Values.imagePullSecrets }}
   imagePullSecrets:
@@ -56,11 +58,9 @@ spec:
         - mountPath: /root/.lnd/tls.cert
           name: config
           subPath: tls.cert
-    {{- if .Values.circuitBreaker }}
-    - name: circuitbreaker
-      image: pinheadmz/circuitbreaker:278737d
-      imagePullPolicy: IfNotPresent
-    {{- end}}
+    {{- with .Values.extraContainers }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   volumes:
     {{- with .Values.volumes }}
       {{- toYaml . | nindent 4 }}

resources/charts/bitcoincore/charts/lnd/templates/service.yaml

@@ -20,5 +20,11 @@ spec:
       targetPort: rest
       protocol: TCP
       name: rest
+{{- if .Values.metricsExport }}
+    - port: {{ .Values.prometheusMetricsPort }}
+      targetPort: prom-metrics
+      protocol: TCP
+      name: prometheus-metrics
+{{- end }}
   selector:
     {{- include "lnd.selectorLabels" . | nindent 4 }}

resources/charts/bitcoincore/charts/lnd/templates/servicemonitor.yaml

@@ -0,0 +1,15 @@
+{{- if .Values.metricsExport }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "lnd.fullname" . }}
+  labels:
+    app.kubernetes.io/name: lnd-metrics
+    release: prometheus
+spec:
+  endpoints:
+    - port: prometheus-metrics
+  selector:
+    matchLabels:
+      app: {{ include "lnd.fullname" . }}
+{{- end }}

resources/plugins/simln/charts/simln/templates/configmap.yaml

@@ -17,5 +17,9 @@ data:
     IQDPofN0fEl5gTwCYhk3nZbjMqJhZ8BsSJ6K8XRhxr7zbwIhAPsgQCFOqUWg632O
     NEO53OQ6CIqnpxSskjsFNH4ZBQOE
     -----END CERTIFICATE-----
-  admin.macaroon.hex: |
-    0201036c6e6402f801030a1062beabbf2a614b112128afa0c0b4fdd61201301a160a0761646472657373120472656164120577726974651a130a04696e666f120472656164120577726974651a170a08696e766f69636573120472656164120577726974651a210a086d616361726f6f6e120867656e6572617465120472656164120577726974651a160a076d657373616765120472656164120577726974651a170a086f6666636861696e120472656164120577726974651a160a076f6e636861696e120472656164120577726974651a140a057065657273120472656164120577726974651a180a067369676e6572120867656e657261746512047265616400000620b17be53e367290871681055d0de15587f6d1cd47d1248fe2662ae27f62cfbdc6
+{{- $configMaps := lookup "v1" "ConfigMap" .Release.Namespace "" }}
+{{- range $configMaps.items }}
+  {{- if and .metadata.labels (hasKey .metadata.labels "role") (eq (index .metadata.labels "role") "macaroon-ref") }}
+  admin.macaroon.hex: {{ index .data "MACAROON_HEX" | quote }}
+  {{- end }}
+{{- end }}

test/data/ln/network.yaml

@@ -2,26 +2,15 @@ nodes:
   - name: tank-0000
     addnode:
       - tank-0001
-    ln:
-      lnd: true
-
   - name: tank-0001
     addnode:
       - tank-0002
-    ln:
-      lnd: true
-
   - name: tank-0002
     addnode:
       - tank-0000
-    ln:
-      lnd: true
-
   - name: tank-0003
     addnode:
       - tank-0000
-    ln:
-      lnd: true
     lnd:
       config: |
         bitcoin.timelockdelta=33
@@ -32,12 +21,9 @@ nodes:
           target: tank-0004-ln
           capacity: 100000
           push_amt: 50000
-
   - name: tank-0004
     addnode:
       - tank-0000
-    ln:
-      lnd: true
     lnd:
       channels:
         - id:
@@ -46,9 +32,6 @@ nodes:
           target: tank-0005-ln
           capacity: 50000
           push_amt: 25000
-
   - name: tank-0005
     addnode:
-      - tank-0000
-    ln:
-      lnd: true
+      - tank-0000

test/data/ln/node-defaults.yaml

@@ -1,8 +1,40 @@
+# enable collectLogs and metricsExport to activate publish lnd-exporter metrics
+
+#Core configs
 image:
   repository: bitcoindevproject/bitcoin
   pullPolicy: IfNotPresent
   tag: "27.0"
+collectLogs: false
+metricsExport: false
 
+#LN configs
+ln:
+  lnd: true
 lnd:
   defaultConfig: |
-    color=#000000
+    color=#000000
+  config: |
+    bitcoin.timelockdelta=33
+  metricsExport: false
+  prometheusMetricsPort: 9332
+  extraContainers:
+    - name: lnd-exporter
+      image: bitdonkey/lnd-exporter:0.1.3
+      imagePullPolicy: IfNotPresent
+      volumeMounts:
+        - name: config
+          mountPath: /macaroon.hex
+          subPath: MACAROON_HEX
+      env:
+        - name: METRICS
+          value: >
+            lnd_balance_channels=parse("/v1/balance/channels","balance") 
+            lnd_local_balance_channels=parse("/v1/balance/channels","local_balance.sat") 
+            lnd_remote_balance_channels=parse("/v1/balance/channels","remote_balance.sat") 
+            lnd_block_height=parse("/v1/getinfo","block_height") 
+            lnd_peers=parse("/v1/getinfo","num_peers")
+      ports:
+        - name: prom-metrics
+          containerPort: 9332
+          protocol: TCP

test/data/logging/network.yaml

@@ -11,5 +11,27 @@ nodes:
   - name: tank-0002
     addnode:
       - tank-0000
+    ln:
+      lnd: true
+    lnd:
+      metricsExport: true
+      prometheusMetricsPort: 9332
+      extraContainers:
+        - name: lnd-exporter
+          image: bitdonkey/lnd-exporter:0.1.3
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: config
+              mountPath: /macaroon.hex
+              subPath: MACAROON_HEX
+          env:
+            - name: METRICS
+              value: >
+                lnd_block_height=parse("/v1/getinfo","block_height")
+                lnd_peers=parse("/v1/getinfo","num_peers")
+          ports:
+            - name: prom-metrics
+              containerPort: 9332
+              protocol: TCP
 caddy:
   enabled: true

test/logging_test.py

@@ -94,6 +94,7 @@ def get_five_values_for_metric(metric):
 
         self.wait_for_predicate(lambda: get_five_values_for_metric("blocks"))
         self.wait_for_predicate(lambda: get_five_values_for_metric("txrate"))
+        self.wait_for_predicate(lambda: get_five_values_for_metric("lnd_block_height"))
 
         # Verify default dashboard exists
         dbs = requests.get(f"{self.grafana_url}/api/search").json()