donations: actual integration test and several fixes/improvements

- integration test doing a donation and checking if it shows up
- production WSGI server waitress instead of dev server
- CLN rpc fixes, decodepay -> decode, wait if listinvoies is not yet populated, msat field name fixed
- qrcode dependency was missing pil to render qr codes
- multithreading was no longer working for some reason, replaced with threading
- fix typo in template copyrite -> copyright
- formatted template code

Author: daywalker90

donations/donations.py

@@ -3,13 +3,14 @@
 # /// script
 # requires-python = ">=3.9.2"
 # dependencies = [
-#   "qrcode>=7.4.2",
+#   "qrcode[pil]>=7.4.2",
 #   "flask>=2.3.3",
 #   "pyln-client>=24.11",
 #   "flask-bootstrap>=3.3.7.1",
 #   "flask-wtf>=1.2.1",
 #   "werkzeug>=3.0.6",
 #   "wtforms>=3.1.2",
+#   "waitress>=3.0.2",
 # ]
 # ///
 
@@ -31,8 +32,10 @@
 """
 
 import base64
-import multiprocessing
 import qrcode
+import threading
+import logging
+import sys
 
 
 from flask import Flask, render_template
@@ -43,6 +46,7 @@
 from random import random
 from wtforms import StringField, SubmitField, IntegerField
 from wtforms.validators import DataRequired, NumberRange
+from waitress.server import create_server
 
 
 plugin = Plugin()
@@ -79,7 +83,10 @@ def make_base64_qr_code(bolt11):
 
 def ajax(label):
     global plugin
-    msg = plugin.rpc.listinvoices(label)["invoices"][0]
+    invoices = plugin.rpc.listinvoices(label)["invoices"]
+    if len(invoices) == 0:
+        return "waiting"
+    msg = invoices[0]
     if msg["status"] == "paid":
         return "Your donation has been received and is well appricated."
     return "waiting"
@@ -105,8 +112,8 @@ def donation_form():
         if invoice["label"].startswith("ln-plugin-donations-"):
             # FIXME: change to paid after debugging
             if invoice["status"] == "paid":
-                bolt11 = plugin.rpc.decodepay(invoice["bolt11"])
-                satoshis = int(bolt11["msatoshi"]) // 1000
+                bolt11 = plugin.rpc.decode(invoice["bolt11"])
+                satoshis = int(bolt11["amount_msat"]) // 1000
                 description = bolt11["description"]
                 ts = bolt11["created_at"]
                 donations.append((ts, satoshis, description))
@@ -126,14 +133,25 @@ def donation_form():
         )
 
 
-def worker(port):
+def worker(port, ready_event):
     app = Flask("donations")
     # FIXME: use hexlified hsm secret or something else
     app.config["SECRET_KEY"] = "you-will-never-guess-this"
     app.add_url_rule("/donation", "donation", donation_form, methods=["GET", "POST"])
     app.add_url_rule("/is_invoice_paid/<label>", "ajax", ajax)
     Bootstrap(app)
-    app.run(host="0.0.0.0", port=port)
+
+    server = create_server(app, host="*", port=port)
+
+    app.logger.setLevel(logging.INFO)
+    logging.getLogger("waitress").setLevel(logging.INFO)
+
+    app.logger.info(f"Starting donation server on port {port} on all addresses")
+
+    jobs[port]["server"] = server
+    ready_event.set()
+
+    server.run()
     return
 
 
@@ -144,21 +162,38 @@ def start_server(port):
     if port in jobs:
         return False, "server already running"
 
-    p = multiprocessing.Process(
-        target=worker, args=[port], name="server on port {}".format(port)
+    ready_event = threading.Event()
+    thread = threading.Thread(
+        target=worker,
+        args=[port, ready_event],
+        name=f"server on port {port}",
+        daemon=False,
     )
-    p.daemon = True
 
-    jobs[port] = p
-    p.start()
+    jobs[port] = {"thread": thread, "ready": ready_event}
+    thread.start()
+
+    ready_event.wait(timeout=5.0)
+    if "server" not in jobs[port]:
+        return False, "server failed to start in time"
 
     return True
 
 
+@plugin.subscribe("shutdown")
+def on_rpc_command_callback(plugin, **kwargs):
+    for port in list(jobs.keys()):
+        stop_server(port)
+    sys.exit()
+
+
 def stop_server(port):
     if port in jobs:
-        jobs[port].terminate()
-        jobs[port].join()
+        server = jobs[port]["server"]
+        thread = jobs[port]["thread"]
+        server.close()
+        if thread.is_alive():
+            thread.join(timeout=2.0)
         del jobs[port]
         return True
     else:

donations/pyproject.toml

@@ -8,13 +8,14 @@ readme = "README.md"
 requires-python = ">=3.9.2"
 
 dependencies = [
-    "qrcode>=7.4.2",
+    "qrcode[pil]>=7.4.2",
     "flask>=2.3.3",
     "pyln-client>=24.11",
     "flask-bootstrap>=3.3.7.1",
     "flask-wtf>=1.2.1",
     "werkzeug>=3.0.6",
     "wtforms>=3.1.2",
+    "waitress>=3.0.2",
 ]
 
 [dependency-groups]
@@ -25,4 +26,5 @@ dev = [
     "pyln-testing>=24.11",
     "pyln-client>=24.11",
     "pyln-proto>=24.11",
+    "requests>=2.32.5",
 ]

donations/templates/donation.html

@@ -2,78 +2,83 @@
 {% import 'bootstrap/wtf.html' as wtf %}
 
 {% block title %}
-    Lightning Donations
+Lightning Donations
 {% endblock %}
 
 {% block navbar %}
-    <nav class="navbar navbar-default">
-        ... navigation bar here (see complete code on GitHub) ...
-    </nav>
+<nav class="navbar navbar-default">
+  ... navigation bar here (see complete code on GitHub) ...
+</nav>
 {% endblock %}
 
 {% block content %}
-    <div class="container">
-      <h1>Leave a donation to support my work!</h1>
-      {% if bolt11 %}
-      <div id="target_div">
-	<div>
-	  <input type="text" value="{{bolt11}}" id="bolt11">
-	  <button onclick="copyFunction()">Copy invoice</button>
-	</div>
-	<div>
-	  <img src="data:image/png;base64,{{qr}}" />
-	</div>
-      </div>
+<div class="container">
+  <h1>Leave a donation to support my work!</h1>
+  {% if bolt11 %}
+  <div id="target_div">
+    <div>
+      <input type="text" value="{{bolt11}}" id="bolt11">
+      <button onclick="copyFunction()">Copy invoice</button>
+    </div>
+    <div>
+      <img src="data:image/png;base64,{{qr}}" />
+    </div>
+  </div>
   {% else %}
-	{{ wtf.quick_form(form) }}
+  {{ wtf.quick_form(form) }}
   {% endif %}
   <h2>Most recent donations & comments</h2>
   <ul>
     {% for item in donations %}
-        <li>{{ item[1] }} Satoshi. Message: {{ item[2] }}</a></li>
+    <li>{{ item[1] }} Satoshi. Message: {{ item[2] }}</a></li>
     {% endfor %}
-</ul>
-<p>The above texts come from a community of unknown users. If you think they violate against your copyrite please <a href="https://www.rene-pickhardt.de/imprint" rel="nofollow"> contact me</a> so that I can remove those comments. According to the German law I am not responsible for Copyright violations rising from user generated content unless you notify me and I don't react.</p>
+  </ul>
+  <p>The above texts come from a community of unknown users. If you think they violate against your copyright please <a
+      href="https://www.rene-pickhardt.de/imprint" rel="nofollow"> contact me</a> so that I can remove those comments.
+    According to the German law I am not responsible for Copyright violations rising from user generated content unless
+    you notify me and I don't react.</p>
 
-<hr>
-<p>
-  c-lightning invoice query service for donations and spontanious payments is brought to you by
-  <a href="https://ln.rene-pickhardt.de">Rene Pickhardt</a>.</p>
+  <hr>
+  <p>
+    c-lightning invoice query service for donations and spontanious payments is brought to you by
+    <a href="https://ln.rene-pickhardt.de">Rene Pickhardt</a>.
+  </p>
 
-<p>
-If you want to learn more about the Lightning network (for beginners and for developers) check out
-  <a href="https://www.youtube.com/user/RenePickhardt">his youtube channel</a>.
-</p>
-<p>
-  Find the source code for this plugin at: <a href="https://github.com/ElementsProject/lightning/tree/master/contrib/plugins/donations">https://github.com/ElementsProject/lightning/tree/master/contrib/plugins/donations</a>
-</p>
+  <p>
+    If you want to learn more about the Lightning network (for beginners and for developers) check out
+    <a href="https://www.youtube.com/user/RenePickhardt">his youtube channel</a>.
+  </p>
+  <p>
+    Find the source code for this plugin at: <a
+      href="https://github.com/ElementsProject/lightning/tree/master/contrib/plugins/donations">https://github.com/ElementsProject/lightning/tree/master/contrib/plugins/donations</a>
+  </p>
 </div>
 {% endblock %}
 {% block scripts %}
 {{super()}}
 <script>
-var interval = null;
-$(document).on('ready',function(){
-    interval = setInterval(updateDiv,3000);
-});
+  var interval = null;
+  $(document).on('ready', function () {
+    interval = setInterval(updateDiv, 3000);
+  });
 
-function updateDiv(){
+  function updateDiv() {
     $.ajax({
-        url: '/is_invoice_paid/{{label}}',
-        success: function(data){
-               if (data != "waiting") {
-                            var tc = document.getElementById("target_div");
-                            tc.innerHTML = data;
-                            clearInterval(interval);
-                        }
+      url: '/is_invoice_paid/{{label}}',
+      success: function (data) {
+        if (data != "waiting") {
+          var tc = document.getElementById("target_div");
+          tc.innerHTML = data;
+          clearInterval(interval);
         }
+      }
     });
-}
+  }
 
-function copyFunction() {
+  function copyFunction() {
     document.getElementById("bolt11").select();
     document.execCommand("copy");
     alert("Copied invoice to clipboard.");
-}
+  }
 </script>
 {% endblock %}

donations/test_donations.py

@@ -1,12 +1,15 @@
 import os
+import re
+import time
+import pytest
 from pyln.testing.fixtures import *  # noqa: F401,F403
-from ephemeral_port_reserve import reserve  # type: ignore
+import requests
 
 plugin_path = os.path.join(os.path.dirname(__file__), "donations.py")
 
 
 def test_donation_starts(node_factory):
-    l1 = node_factory.get_node(allow_warning=True)
+    l1 = node_factory.get_node()
     # Test dynamically
     l1.rpc.plugin_start(plugin_path)
     l1.rpc.plugin_stop(plugin_path)
@@ -19,10 +22,61 @@ def test_donation_starts(node_factory):
 
 def test_donation_server(node_factory):
     pluginopt = {"plugin": plugin_path, "donations-autostart": False}
-    l1 = node_factory.get_node(options=pluginopt, allow_warning=True)
-    port = reserve()
+    l1 = node_factory.get_node(options=pluginopt)
+    port = node_factory.get_unused_port()
     l1.rpc.donationserver("start", port)
-    l1.daemon.wait_for_log("plugin-donations.py:.*Serving Flask app 'donations'")
-    l1.daemon.wait_for_log("plugin-donations.py:.*Running on all addresses")
+    l1.daemon.wait_for_log(
+        f"plugin-donations.py:.*Starting donation server on port {port} on all addresses"
+    )
+
+    session = requests.Session()
+
+    response = session.get(f"http://127.0.0.1:{port}/donation")
+    assert response.status_code == 200
+    assert "Leave a donation" in response.text
+
+    match = re.search(r'name="csrf_token"[^>]*value="([^"]+)"', response.text)
+    assert match, "Could not find CSRF token"
+    csrf_token = match.group(1)
+
+    response = session.post(
+        f"http://127.0.0.1:{port}/donation",
+        data={
+            "csrf_token": csrf_token,
+            "amount": "1000",
+            "description": "Test donation from pytest",
+        },
+    )
+    assert response.status_code == 200
+    assert "The CSRF token is missing" not in response.text
+
+    assert "data:image/png" in response.text
+
+    match = re.search(r'value="(lnbc[^"]+)"', response.text)
+    assert match, "No bolt11 invoice found in response"
+    bolt11 = match.group(1)
+    l1.rpc.call("xpay", [bolt11])
+    label = l1.rpc.call("listinvoices", {"invstring": bolt11})["invoices"][0]["label"]
+
+    max_attempts = 10
+    for attempt in range(max_attempts):
+        response = session.get(f"http://127.0.0.1:{port}/is_invoice_paid/{label}")
+
+        if response.text != "waiting":
+            # Payment detected!
+            assert (
+                "Your donation has been received and is well appricated."
+                in response.text
+            )
+            break
+
+        time.sleep(1)
+    else:
+        pytest.fail("Payment was not detected after 10 seconds")
+
+    response = session.get(f"http://127.0.0.1:{port}/donation")
+    assert "1000 Satoshi" in response.text
+    assert "Test donation from pytest" in response.text
+
     msg = l1.rpc.donationserver("stop", port)
     assert msg == f"stopped server on port {port}"