Merge branch 'master' into guilt/fix-flakes12

Author: rustyrussell

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,5 +1,5 @@
 > [!IMPORTANT]
-> 24.11 FREEZE NOVEMBER 7TH: Non-bugfix PRs not ready by this date will wait for 25.02.
+> 25.02 FREEZE JANUARY 31ST: Non-bugfix PRs not ready by this date will wait for 25.05.
 
 ## Checklist
 Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:

.github/workflows/repro.yml

@@ -19,42 +19,48 @@ jobs:
       - name: Git checkout
         uses: actions/checkout@v4
 
-      - name: Build environment setup
+      - name: Build environment setup - ${{ matrix.version }}
         run: |
           echo "Building base image for ${{ matrix.version }}"
+          echo "STEP=Build environment setup" >> "$GITHUB_ENV"
           sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
-              apt-get update && \
-              apt-get install -y debootstrap && \
-              debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
+            apt-get update && \
+            apt-get install -y debootstrap && \
+            debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
           sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
 
-      - name: Builder image setup
-        run: docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
+      - name: Builder image setup - ${{ matrix.version }}
+        run: |
+          echo "STEP=Builder image setup" >> "$GITHUB_ENV"
+          docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
 
-      - name: Build using the builder image and store Git state
+      - name: Build reproducible image and store Git state - ${{ matrix.version }}
         run: |
+          echo "STEP=Build reproducible image and store Git state" >> "$GITHUB_ENV"
+
           # Create release directory.
           mkdir $GITHUB_WORKSPACE/release
 
           # Perform the repro build.
           docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
 
           # Commit the image in order to inspect the build later.
-          docker commit cl-build cl-release
+          docker commit cl-build cl-repro
 
           # Inspect the version.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-release bash -c "make version > /repo/release/version.txt"
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "make version > /repo/release/version.txt"
 
           # Inspect the Git tree state.
-          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-release bash -c "\
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-repro bash -c "\
               git --no-pager status > /repo/release/git.log && \
               git --no-pager diff >> /repo/release/git.log"
 
           # Change permissions on the release files for access by the runner environment.
           sudo chown -R runner $GITHUB_WORKSPACE/release
 
-      - name: Assert clean version and release
+      - name: Assert clean version - ${{ matrix.version }}
         run: |
+          echo "STEP=Assert clean version" >> "$GITHUB_ENV"
           echo 'Version:'
           cat release/version.txt
           echo -e
@@ -63,16 +69,45 @@ jobs:
           echo 'Release file:'
           ls -al release/clightning-*
           echo -e
-
           if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
-             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]
-          then
+             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]; then
             echo "Git Status and Diff:"
             cat release/git.log
             echo -e
-
             echo 'Error: release modded / dirty tree.'
             exit 1
           else
             echo 'Success! Clean release.'
           fi
+
+      - name: Upload release artifact - ${{ matrix.version }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: release-${{ matrix.version }}
+          path: release
+          retention-days: 3 # Automatically delete after 3 days
+
+      - name: Send email on failure
+        if: ${{ failure() }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 587
+          username: ${{ secrets.EMAIL_USERNAME }}
+          password: ${{ secrets.EMAIL_PASSWORD }}
+          from: ${{ secrets.EMAIL_USERNAME }}
+          to: ${{ vars.DISTRIBUTION_LIST }}
+          subject: "CI Failure: Step ${{ env.STEP }} failed for distro ${{ matrix.version }}"
+          convert_markdown: true
+          html_body: |
+            <html>
+              <body>
+                <p>GitHub Workflow ${{ github.workflow }} Failed! For more details, click on the action below.</p>
+                <strong>Failure Details:</strong><br/>
+                <strong>Event: </strong>${{ github.event_name }}<br/>
+                <strong>Job: </strong>${{ github.job }}<br/>
+                <strong>Distro: </strong>${{ matrix.version }}<br/>
+                <strong>Step: </strong>${{ env.STEP }}<br/>
+                <strong>Action: </strong><a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}</a><br/>
+              </body>
+            </html>

contrib/keys/cln.txt

@@ -0,0 +1,52 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGdhz1QBEADLRZWvhey6GDkcn/VyKTB+v7d8hn/ElwQXlCA7sU0xodwHtUDE
+W3jsjWhLPiIgd/QIPfFQF6NKIXNTgNgTs0je5n0rgK1VaFf19aIRa6nY5qIDGWKS
+DONkdi8UoGcBygq2aBIz+uDcsBD9z9iyQ239Pnj0eAcKf9M0fvCXT5LPkI8Nzx35
+gX9cpkA97tjfMSNeRcXkqXNoYWNx3UGCM5vMf6MUAz8OHpvIxXYVBp8yV+SZrF0n
+zLcqYDMvl9U1R6NqSnCScEkIwDzFB3PjQXRAIhlOmfuyl5Q7RWEv9rg2YeKiOUgX
+m1I+HdvEHEpqwqr4d70H9KXAUPtdQXuxxP5to2gFwwe7JQALXY12L2L1impieZdF
+AgDzD7EY1oFudqEYtW03gVzABf3nqscBB9xI905sjhvvPXkgP1hv09Ar1qsULDdx
+o8P8N5LVMh8cQUfWf3f1pTwSizimiAxdljpvOu6lkntO3OWv8ql5V5yEpm1KMREV
+n5GBlbsy8x0wUx0x7b5UN0JhxvwPbBzufYHW36r4kZUbAsimzeAL1FpUA5mP4ZQU
+A3+A9LwhL2BBmGeQaW77pl0mO3/ocFv5nwlt8hoK6744mWlgrHgeEGGiomu4H+i7
+IFzKFU5FXv5Ych2fUIb/KgiDRG5jviN+4QL7iJ2XrbXtC5n2HVFG+mu9gQARAQAB
+tCpDb3JlIExpZ2h0bmluZyAoQ0xOKSA8Y2xuQGJsb2Nrc3RyZWFtLmNvbT6JAlQE
+EwEKAD4WIQRhbFL5nQYSsqFRsQdBKamUqn6YUgUCZ2HPVAIbAwUJBaOagAULCQgH
+AgYVCgkICwIEFgIDAQIeAQIXgAAKCRBBKamUqn6YUnPvD/wI0XwTkrgltDCgAwPh
+iNZZTjHjSglIsfFHsLArkVstfbcwhoH+K9nziKy2WsOxsH19JL5fHcvutaNFCdQW
+7QUA4tWMB4HDY1y2JcPIPFfrDHFEoWajvdao/qWc0IKk/538VvpjCfWrdrXGhSi3
+5+E0U7VUBSb66SLtVRtH8EznfJ/9phDGTr78ZFFOZoxoBm7Fb1txl+E8j6HRd7Uc
+cb4V+q36m0wLjeIB1yyupHVu77iRjXFg+b8rV/7jaGuLwmxQzxgLLx1vQQf7N6SA
+wltWEzZ4ZJ6cBYI2Lz9/CCyYxta6q0HQFBvDGod2uz6SLuTFCeviVl05r9pglqA+
+E6ZmPDf42ieCEpTAEi5k54rs2jh0Rtk0cE8Lo9SgaJgGumKWH64deVH/TKGn0urP
+SiULKqr/O7/IC50zT//Rk/amtJWaltNEYglkpa61eZvyqltH54CAsR/5g9oM2nLD
+vEQiNhZ8ZI2ixlllYBdIrmj3n6NejoQ9W1YaxaMzDCW7QaQnPdXnhrLiY1i3q6OW
+bcJviiE5etNc/cWDy2Hg9xU3d6YjC0kp/KdxF1Yfl+7rvCna66kSPzwWWgB7PIkn
+lTdvIGqBG4fhH30BAdETwCcX4Ou3Rtcc88fzkZ3PGlRhguP1W4mwrxyGIiuP02Pl
+QxyDPpEbhwD47XGug9Pjrevjh7kCDQRnYc9UARAAtomSOFZoWVCDybTSyNtbkq32
+nw2FgZnMkXSnnhTjsvdlnh/kP5OfixLMrXN9BhYZM931mHpOhPDckZfoy25UnKbQ
+yIbeOkY2TwCA2ZNXU31Va5UU4o6Jod8RbrfdxezgQpU98GjQxkABoaXIFuXbAcln
+yPyVGSvDKDlf2lZdzWapKcc6DWwPWu+hvUALH0AbdgsZbUjBmkpUoSQ5Wpwzj3HT
+eoOz6ZUepKRZ7wQDrJLYxb4yqsf8OpoNOVh2Zz2h/p3zTvm3uoBs6ORFDusfp5Rz
+kESKNMKYfNZDhfSp9IsHdWoUMLbv+le/McM5ZtUDdgcHGw36MWjQMywgOmKUC7y2
+iyZkQVHw6HOTur1ou0EN/a2ANRfI0x8XLP1Uj4RsaY+bBzmQUUq/t+ioInlbzYuE
+nmfPek+eAVVuH/Q8asKCYZlVy2o50hCJojXcjXK6bax8PGoWpvCZFGDTDeXVLsB9
+F7KpkPcYBGY+0lkzcLAqr873EJGEXrJUZw9M3bYF6yQuM9b6IMZ5FQsQVuZ1ctZd
+MJPDdHTsweh6VksoZ+8OdhRU5geXj83psgYvgHAIOxMxgZ7Gh8pVtWvhTwPieGFK
+mOfkoJIjgKAV7IA2DqSz8P4/97mM2ex4II16Q2woHA6/a9VYvj+xc9+NgP4ZlimB
+VfLvB4dK0xRCwhkp4a8AEQEAAYkCPAQYAQoAJhYhBGFsUvmdBhKyoVGxB0EpqZSq
+fphSBQJnYc9UAhsMBQkFo5qAAAoJEEEpqZSqfphSBrcP/09drXoIA2i8vblntJwr
+SNYjdw16tU1uE1WvYXX1fbTH8+6+p+h8lc4hAtM3FyEINQ37uQ3gz49PusNG3lNT
+L/jcpEskmQCWTsgJlv5b03O7y+vRb5aEuc9Vn4iCA4N7487ICvJsPDp7cGBxthX0
+YkDDZ9+bOs3dym1MVZQo4tJiQuPvux2M3dk6WUHOUnKu0G/fHCB/a4q5DW/G6q22
+mndUTrsUrEe0xdmfDiNbuh2xHNNzQCaNlh0yzWLm6uREaOnzv3Ys/F/+2IhCehmg
+0jUl1BIZBJYR0ZXRK2pBY4d+pkUoOV/6oqgadwgP1VT3pBrC/Uo1bZXrxfFwgzgj
+rQJt2C9vJoz+VZLAQeZcgamlzlgE+L7QbfbJ9xjPlY9YE2Tifj5QfVxy70pfHnOY
++4QEa8QSXIew6dEzpAWX8OGZ/F42EgkS3Ogypct19ob0g4n4NuVQ264NsuxvTqaV
+cT2vX4OkHie82v5N/b7py4/GuxRCc5+Mz015LWYkxUbpD+wAAU0H/ZPIrL24wUW+
+aFr+3oTb8YOCz58NUzClFDvZPCu9jDUzYBcJH0HSEe5BTh2eaM9HaUJ5Q6Xs4/Iv
+GiPFL/CaqgMQHDis5WjGZwhmIwbxyuL9EWxbK8kZob1iVTL5fD4YZgsqjWrDSbed
+Uv36UtFhm/IcjGXnG72mhq1W
+=jLDp
+-----END PGP PUBLIC KEY BLOCK-----

doc/contribute-to-core-lightning/coding-style-guidelines/writing-json-schemas.md

@@ -94,7 +94,7 @@ preventing accidental overwrites from unrelated tests.
 	where `n` can be any number of repetitions. OR by manually running the test multiple times with:
 
 	```bash
-	rm -rf /tmp/ltests* && make -s && VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py
+	rm -rf /tmp/ltests* && make -s && VALGRIND=0 TIMEOUT=40 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv -s tests/autogenerate-rpc-examples.py
 	```
 
 	- Identify changing values, and add them to `REPLACE_RESPONSE_VALUES`:
@@ -117,7 +117,7 @@ environment variable with a comma-separated list of method names. Eg. `REGENERAT
 only regenerate examples for the `getinfo` and `connect` RPCs.
 2. To regenerate specific examples, set the REGENERATE environment variable:
 ```bash
-REGENERATE='getinfo,connect' VALGRIND=0 TIMEOUT=10 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv tests/autogenerate-rpc-examples.py
+REGENERATE='getinfo,connect' VALGRIND=0 TIMEOUT=10 TEST_DEBUG=1 GENERATE_EXAMPLES=1 pytest -vvv -s tests/autogenerate-rpc-examples.py
 ```
 3. Logs are saved in `tests/autogenerate-examples-status.log`, and JSON data is in `tests/autogenerate-examples.json`.
 4. Run `make` after the script completes to ensure schema updates are applied in other places too, such as `...msggen/schema.json`.
@@ -127,4 +127,4 @@ REGENERATE='getinfo,connect' VALGRIND=0 TIMEOUT=10 TEST_DEBUG=1 GENERATE_EXAMPLE
 
 1. Sip whenever you have an additional comma at the end of a sequence.
 2. Sip whenever you omit a comma in a sequence because you cut & paste.
-3. Skull whenever you wish JSON had comments.
+3. Skull whenever you wish JSON had comments.

tests/autogenerate-rpc-examples.py

@@ -297,15 +297,6 @@
 
 if os.path.exists(LOG_FILE):
     open(LOG_FILE, 'w').close()
-
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s - %(levelname)s - %(message)s',
-                    datefmt='%H:%M:%S',
-                    handlers=[
-                        logging.FileHandler(LOG_FILE),
-                        logging.StreamHandler()
-                    ])
-
 logger = logging.getLogger(__name__)
 
 
@@ -2035,6 +2026,19 @@ def generate_list_examples(l1, l2, l3, c12, c23_2, inv_l31, inv_l32, offer_l23,
         raise
 
 
+@pytest.fixture(autouse=True)
+def setup_logging():
+    global logger
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s", "%H:%M:%S")
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+    file_handler = logging.FileHandler(LOG_FILE)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+
 @unittest.skipIf(not GENERATE_EXAMPLES, 'Generates examples for doc/schema/lightning-*.json files.')
 def test_generate_examples(node_factory, bitcoind, executor):
     """Re-generates examples for doc/schema/lightning-*.json files"""
@@ -2055,9 +2059,9 @@ def list_all_examples():
                 for node in ast.walk(tree):
                     if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'update_example':
                         for keyword in node.keywords:
-                            if (keyword.arg == 'method' and isinstance(keyword.value, ast.Str)):
-                                if keyword.value.s not in methods:
-                                    methods.append(keyword.value.s)
+                            if (keyword.arg == 'method' and isinstance(keyword.value, ast.Constant)):
+                                if keyword.value.value not in methods:
+                                    methods.append(keyword.value.value)
                 return methods
             except Exception as e:
                 logger.error(f'Error in listing all examples: {e}')
@@ -2105,5 +2109,5 @@ def list_missing_examples():
         update_examples_in_schema_files()
         logger.info('All Done!!!')
     except Exception as e:
-        logger.error(e)
+        logger.error(e, exc_info=True)
         sys.exit(1)