Merge branch 'master' into guilt/wallet-can-spend-takes-ages

Author: rustyrussell

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 > [!IMPORTANT]
-> 24.11 FREEZE NOVEMBER 7TH: Non-bugfix PRs not ready by this date will wait for 25.02.
+> 25.02 FREEZE JANUARY 31ST: Non-bugfix PRs not ready by this date will wait for 25.05.
 
 ## Checklist
 Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:

.github/scripts/setup.sh

@@ -20,12 +20,18 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      gcc-arm-none-eabi \
      gettext \
      git \
+     gnupg \
      jq \
      libc6-dev-arm64-cross \
      libc6-dev-armhf-cross \
+     libev-dev \
+     libevent-dev \
+     libffi-dev \
+     libicu-dev \
      libpq-dev \
      libprotobuf-c-dev \
      libsqlite3-dev \
+     libssl-dev \
      libtool \
      libxml2-utils \
      locales \
@@ -43,6 +49,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \
      software-properties-common \
      sudo \
      tcl \
+     tclsh \
      unzip \
      valgrind \
      wget \

.github/scripts/sync-rpc-cmds.py

@@ -70,7 +70,7 @@ def extract_rpc_commands(rst_content):
         commands = re.findall(
             r"\b([a-zA-Z0-9_-]+)" r"\s+<([^>]+)>\n", manpages_block.group(1)
         )
-        return commands
+        return [(re.sub(r"\blightning-", "", rpc_name), file_name) for rpc_name, file_name in commands]
     return []
 
 
@@ -102,7 +102,6 @@ def main():
     if commands_from_local:
         order = 0
         for name, file in commands_from_local:
-            # print(f"{name}\t\t{file}")
             with open("doc/" + file) as f:
                 body = f.read()
                 publishDoc(Action.ADD if name in commands_to_add else Action.UPDATE, name, body, order, headers)

.github/workflows/ci.yaml

@@ -88,7 +88,6 @@ jobs:
       - name: Check source
         env:
           VALGRIND: 0
-          GENERATE_EXAMPLES: 1
           PYTEST_OPTS: --timeout=1200
         run: make check-source BASE_REF="origin/${{ github.base_ref }}"
       - name: Check Generated Files have been updated
@@ -480,6 +479,48 @@ jobs:
         run: |
           poetry run pytest tests/ -vvv -n 2 ${PYTEST_OPTS}  ${{ matrix.PYTEST_OPTS }}
 
+  update-docs-examples:
+    name: Update examples in doc schemas 
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: true
+    env:
+      VALGRIND: 0
+      GENERATE_EXAMPLES: 1
+      PYTEST_OPTS: --timeout=1200
+      TEST_NETWORK: regtest
+    needs:
+      - compile
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          bash -x .github/scripts/setup.sh
+          pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
+          poetry export -o /tmp/requirements.txt --without-hashes --with dev
+          pip install -r /tmp/requirements.txt
+      - name: Install bitcoind
+        env:
+          TEST_NETWORK: regtest
+        run: .github/scripts/install-bitcoind.sh
+      - name: Download build
+        uses: actions/download-artifact@v4
+        with:
+          name: cln-compile-gcc.tar.bz2
+      - name: Unpack pre-built CLN
+        run: |
+          tar -xaf cln-compile-gcc.tar.bz2
+      - name: Test
+        run: |
+          make -j $(nproc) check-doc-examples
+
   min-btc-support:
     name: Test minimum supported BTC v${{ matrix.MIN_BTC_VERSION }} with ${{ matrix.NAME }}
     runs-on: ubuntu-22.04
@@ -582,6 +623,7 @@ jobs:
       - check-units
       - integration-valgrind
       - integration-sanitizers
+      - update-docs-examples
       - min-btc-support
       - check-flake
     steps:

.github/workflows/release.yml

@@ -60,7 +60,7 @@ jobs:
       fail-fast: false    # Let each build finish.
       matrix:
         target:
-          - 'bin-Fedora-28-amd64'
+          - 'bin-Fedora'
           - 'bin-Ubuntu-focal'
           - 'bin-Ubuntu-jammy'
           - 'bin-Ubuntu-noble'

.github/workflows/repro.yml

@@ -19,42 +19,48 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v4
 
-      - name: Build environment setup
+      - name: Build environment setup - ${{ matrix.version }}
         run: |
           echo "Building base image for ${{ matrix.version }}"
+          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
           sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
-              apt-get update && \
-              apt-get install -y debootstrap && \
-              debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
+            apt-get update && \
+            apt-get install -y debootstrap && \
+            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
           sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
 
-      - name: Builder image setup
-        run: docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
+      - name: Builder image setup - ${{ matrix.version }}
+        run: |
+          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
+          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
 
-      - name: Build using the builder image and store Git state
+      - name: Build reproducible image and store Git state - ${{ matrix.version }}
         run: |
+          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
+
           # Create release directory.
           mkdir $GITHUB_WORKSPACE/release
 
           # Perform the repro build.
           docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
 
           # Commit the image in order to inspect the build later.
-          docker commit cl-build cl-release
+          docker commit cl-build cl-repro
 
           # Inspect the version.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-release bash -c "make version > /repo/release/version.txt"
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
 
           # Inspect the Git tree state.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-release bash -c "\
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
               git --no-pager status > /repo/release/git.log && \
               git --no-pager diff >> /repo/release/git.log"
 
           # Change permissions on the release files for access by the runner environment.
           sudo chown -R runner $GITHUB_WORKSPACE/release
 
-      - name: Assert clean version and release
+      - name: Assert clean version - ${{ matrix.version }}
         run: |
+          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
           echo 'Version:'
           cat release/version.txt
           echo -e
@@ -63,16 +69,45 @@ jobs:
           echo 'Release file:'
           ls -al release/clightning-*
           echo -e
-
           if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
-             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]
-          then
+             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
             echo "Git Status and Diff:"
             cat release/git.log
             echo -e
-
             echo 'Error: release modded / dirty tree.'
             exit 1
           else
             echo 'Success! Clean release.'
           fi
+
+      - name: Upload release artifact - ${{ matrix.version }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ matrix.version }}
+          path: release
+          retention-days: 3 # Automatically delete after 3 days
+
+      - name: Send email on failure
+        if: ${{ failure() }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 587
+          username: ${{ secrets.EMAIL_USERNAME }}
+          password: ${{ secrets.EMAIL_PASSWORD }}
+          from: ${{ secrets.EMAIL_USERNAME }}
+          to: ${{ vars.DISTRIBUTION_LIST }}
+          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
+          convert_markdown: true
+          html_body: |
+            <html>
+              <body>
+                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
+                <strong>Failure Details:</strong><br/>
+                <strong>Event: </strong>${{ github.event_name }}<br/>
+                <strong>Job: </strong>${{ github.job }}<br/>
+                <strong>Distro: </strong>${{ matrix.version }}<br/>
+                <strong>Step: </strong>${{ env.STEP }}<br/>
+                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
+              </body>
+            </html>

.gitignore

@@ -82,6 +82,7 @@ compile_commands.json
 # Rust targets
 target
 plugins/cln-grpc
+plugins/clnrest
 
 # Build directories
 bionic/

.msggen.json

@@ -997,6 +997,7 @@
         },
         "BlacklistruneRequest": {
             "BlacklistRune.end": 2,
+            "BlacklistRune.relist": 3,
             "BlacklistRune.start": 1
         },
         "BlacklistruneResponse": {
@@ -1772,6 +1773,7 @@
             "GetRoutes.final_cltv": 7,
             "GetRoutes.finalcltv": 6,
             "GetRoutes.layers[]": 4,
+            "GetRoutes.maxdelay": 8,
             "GetRoutes.maxfee_msat": 5,
             "GetRoutes.source": 1
         },
@@ -3728,6 +3730,7 @@
             "Xpay.amount_msat": 2,
             "Xpay.invstring": 1,
             "Xpay.layers[]": 4,
+            "Xpay.maxdelay": 7,
             "Xpay.maxfee": 3,
             "Xpay.partial_msat": 6,
             "Xpay.retry_for": 5
@@ -5125,6 +5128,10 @@
             "added": "pre-v0.10.1",
             "deprecated": null
         },
+        "BlacklistRune.relist": {
+            "added": "v25.02",
+            "deprecated": null
+        },
         "BlacklistRune.start": {
             "added": "pre-v0.10.1",
             "deprecated": null
@@ -7317,6 +7324,10 @@
             "added": "v24.08",
             "deprecated": null
         },
+        "GetRoutes.maxdelay": {
+            "added": "v25.02",
+            "deprecated": null
+        },
         "GetRoutes.maxfee_msat": {
             "added": "v24.08",
             "deprecated": null
@@ -12969,6 +12980,10 @@
             "added": "v24.11",
             "deprecated": null
         },
+        "Xpay.maxdelay": {
+            "added": "v25.02",
+            "deprecated": null
+        },
         "Xpay.maxfee": {
             "added": "v24.11",
             "deprecated": null