diff --git a/Cargo.lock b/Cargo.lock index 6582b976f554..64ecac495114 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3125,8 +3125,7 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "utoipa" version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" +source = "git+https://github.com/rustyrussell/utoipa.git#26f4908b092a4a746cbec935e05165852ea01289" dependencies = [ "indexmap 2.10.0", "serde", @@ -3137,8 +3136,7 @@ dependencies = [ [[package]] name = "utoipa-gen" version = "5.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d79d08d92ab8af4c5e8a6da20c47ae3f61a0f1dabc1997cdf2d082b757ca08b" +source = "git+https://github.com/rustyrussell/utoipa.git#26f4908b092a4a746cbec935e05165852ea01289" dependencies = [ "proc-macro2", "quote", @@ -3149,8 +3147,7 @@ dependencies = [ [[package]] name = "utoipa-swagger-ui" version = "9.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55" +source = "git+https://github.com/rustyrussell/utoipa.git#26f4908b092a4a746cbec935e05165852ea01289" dependencies = [ "axum 0.8.4", "base64 0.22.1", @@ -3168,8 +3165,7 @@ dependencies = [ [[package]] name = "utoipa-swagger-ui-vendored" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" +source = "git+https://github.com/rustyrussell/utoipa.git#26f4908b092a4a746cbec935e05165852ea01289" [[package]] name = "valuable" diff --git a/doc/contribute-to-core-lightning/release-checklist.md b/doc/contribute-to-core-lightning/release-checklist.md index fd91889dcb69..24716f218f3e 100644 --- a/doc/contribute-to-core-lightning/release-checklist.md +++ b/doc/contribute-to-core-lightning/release-checklist.md @@ -24,18 +24,19 @@ Here's a checklist for the release process. 2. Use `devtools/changelog.py` to collect the changelog entries from pull request commit messages and merge them into the manually maintained `CHANGELOG.md`. This does API queries to GitHub, which are severely ratelimited unless you use an API token: set the `GH_TOKEN` environment variable to a Personal Access Token from 3. Create a new CHANGELOG.md heading to `vrc1`, and create a link at the bottom. Note that you should exactly copy the date and name format from a previous release, as the `build-release.sh` script relies on this. -4. Update the package versions: `make update-versions NEW_VERSION=rc1` +4. Update the package versions: `make update-versions NEW_VERSION=vrc1` 5. Create a PR with the above. ## Releasing -rc1 1. Merge the above PR. 2. Tag it `git pull && git tag -s vrc1`. Note that you should get a prompt to give this tag a 'message'. Make sure you fill this in. -3. Confirm that the tag will show up for builds with `git describe` -4. Push the tag to remote `git push --tags` (pushing the tag will kickoff the "Release 🚀" CI action which builds the release targets and a draft release). -7. Run the script `contrib/cl-repro.sh` for [Builder image setup](https://docs.corelightning.org/docs/repro#builder-image-setup). This will create the required builder images `cl-repro-` for the next step. -8. Sign the release locally by running `tools/build-release.sh --without-zip sign` which will sign the release contents and create SHA256SUMS and SHA256SUMS.asc in the release folder. Compare these with `c-lightning-`.zip on GitHub. -9. Check the generated draft `vrc1` release on Github and check `Set as a pre-release` option. Add the SHA256SUMS.asc from your local release folder to newly drafted release, replacing it. +3. Confirm that the tag will show up for builds with `git describe`. We don't push it to GitHub yet, just in case the following steps fail, and more fixes are required! +5. Run the script `contrib/cl-repro.sh` for [Builder image setup](https://docs.corelightning.org/docs/repro#builder-image-setup). This will create the required builder images `cl-repro-` for the next step. +6. Sign the release locally by running `tools/build-release.sh` which will sign the release contents and create `SHA256SUMS-v` and `SHA256SUMS-v.asc` in the release folder. +7. Push the tag to remote `git push --tags` (pushing the tag will kickoff the "Release 🚀" CI action which builds the release targets and a draft release). +8. Compare your release/`c-lightning-`.zip on GitHub. +9. Check the generated draft `vrc1` release on Github and check `Set as a pre-release` option. Add the `SHA256SUMS-v.asc` from your local release folder to newly drafted release, replacing it. 9. Announce rc1 release on core-lightning's release-chat channel on Discord & [BuildOnL2](https://community.corelightning.org/c/general-questions/). 10. Use `devtools/credit --verbose v` to get commits, days and contributors data for release note. 11. Prepare release notes draft including information from above step, and share with the team for editing. @@ -46,10 +47,10 @@ Here's a checklist for the release process. ## Releasing -rc2, ..., -rcN 1. Update CHANGELOG.md by changing rc(N-1) to rcN. Update the changelog list with information from newly merged PRs also. -2. Update the package versions: `make update-versions NEW_VERSION=rcN` +2. Update the package versions: `make update-versions NEW_VERSION=vrcN` 3. Add a PR with the rcN. 4. Tag it `git pull && git tag -s vrcN && git push --tags` -5. Draft a new `vrcN` pre-release on Github, upload reproducible builds, SHA256SUMS and SHA256SUMS.asc. +5. Draft a new `vrcN` pre-release on Github, upload reproducible builds, `SHA256SUMS-v` and `SHA256SUMS-v.asc`. 5. Announce tagged rc release on core-lightning's release-chat channel on Discord & [BuildOnL2](https://community.corelightning.org/c/general-questions/). 6. Upgrade your personal nodes to the rcN. 7. Confirm that Github actions for PyPI and Docker publishing are working as expected. @@ -57,31 +58,29 @@ Here's a checklist for the release process. ## Tagging the Release 1. Update the CHANGELOG.md; remove -rcN in both places, update the date and add title and namer. -2. Update the contrib/pyln package versions: `make update-versions NEW_VERSION=` +2. Update the contrib/pyln package versions: `make update-versions NEW_VERSION=v` 3. Add a PR with that release. 4. Merge the PR, then: - - `export VERSION=23.05` - `git pull` - - `git tag -a -s v${VERSION} -m v${VERSION}` + - `VERSION=23.05; git tag -a -s v$VERSION -m v$VERSION` - `git push --tags` -5. Run `tools/build-release.sh` to: +5. Run `tools/build-release.sh` (with `--sudo` if you need root to run Docker) to: - Create reproducible zipfile - Build non-reproducible Fedora image - - Build reproducible Ubuntu-v18.04, Ubuntu-v20.04, Ubuntu-v22.04 images. Follow [link](https://docs.corelightning.org/docs/repro#building-using-the-builder-image) for manually Building Ubuntu Images. + - Build reproducible Ubuntu-v20.04, Ubuntu-v22.04 and Ubuntu-v24.04 images. Follow [link](https://docs.corelightning.org/docs/repro#building-using-the-builder-image) for manually Building Ubuntu Images. - Build Docker images for amd64 and arm64v8. Follow [link](https://docs.corelightning.org/docs/docker-images) for more details on Docker publishing. - Create and sign checksums. Follow [link](https://docs.corelightning.org/docs/repro#co-signing-the-release-manifest) for manually signing the release. -6. The tarballs may be owned by root, so revert ownership if necessary: +6. If you used `--sudo`, the tarballs may be owned by root, so revert ownership if necessary: `sudo chown ${USER}:${USER} *${VERSION}*` 7. Upload the resulting files to github and save as a draft. () -8. Send `SHA256SUMS` & `SHA256SUMS.asc` files to the rest of the team to check and sign the release. +8. Send `SHA256SUMS-v` & `SHA256SUMS-v.asc` files to the rest of the team to check and sign the release. 9. Team members can verify the release with the help of `build-release.sh`: - 1. Rename release captain's `SHA256SUMS` to `SHA256SUMS-v${VERSION}` and `SHA256SUMS.asc` to `SHA256SUMS-v${VERSION}.asc`. - 2. Copy them in the root folder (`lightning`). + 1. Copy the release captain's `SHA256SUMS-v` and `SHA256SUMS-v.asc` into the root folder (`lightning`). 3. Run `tools/build-release.sh --verify`. It will create reproducible images, verify checksums and sign. - 4. Send your signatures from `release/SHA256SUMS.new` to release captain. + 4. Send your signatures from `release/SHA256SUMS-v.asc` to release captain. 5. Or follow [link](https://docs.corelightning.org/docs/repro#verifying-a-reproducible-build) for manual verification instructions. -10. Append signatures shared by the team into the `SHA256SUMS.asc` file, verify with `gpg --verify SHA256SUMS.asc` and include the file in the draft release. +10. Append signatures shared by the team into the `SHA256SUMS-v.asc` file, verify with `gpg --verify SHA256SUMS-v.asc` and include the file in the draft release. 11. The GitHub action `Publish Python 🐍 distributions 📦 to PyPI and TestPyPI` should upload the pyln modules to pypi.org. However, this can also be done manually by running `make pyln-release`. This process requires keys for each of the `pyln-client`, `pyln-proto`, and `pyln-testing` modules to be accessible to uv. You can set the key as an environment variable and build and publish each pyln release independently: - `export UV_PUBLISH_TOKEN=` - `make pyln-release-client` @@ -91,7 +90,7 @@ Here's a checklist for the release process. ## Performing the Release -1. Edit the GitHub draft and include the `SHA256SUMS.asc` file. +1. Edit the GitHub draft and include the `SHA256SUMS-v.asc` file. 2. Publish the release as not a draft. 3. Announce the final release on core-lightning's release-chat channel on Discord & [BuildOnL2](https://community.corelightning.org/c/general-questions/). 4. Send a mail to c-lightning and lightning-dev mailing lists, using the same wording as the Release Notes in GitHub. @@ -99,7 +98,6 @@ Here's a checklist for the release process. ## Post-release -1. Wait for a week to see if we need any point releases! 2. Create a PR to update Makefile's CLN_NEXT_VERSION and important dates for the next release on `.github/PULL_REQUEST_TEMPLATE.md`. 3. Look through PRs which were delayed for release and merge them. 4. Close out the Milestone for the now-shipped release. @@ -118,9 +116,9 @@ Here's a checklist for the release process. 9. Create a new release draft for `v.` on GitHub, ensuring to check the `Set as a pre-release` option. 10. Execute the script contrib/cl-repro.sh for the [Builder image setup](https://docs.corelightning.org/docs/repro#builder-image-setup). This will generate the builder images `cl-repro-` needed for the next step. 11. Run the following script to prepare the required builds `tools/build-release.sh bin-Fedora bin-Ubuntu sign`. -12. Upload the reproducible builds along with `SHA256SUMS` and `SHA256SUMS.asc` files from the release folder to the newly drafted release. -13. Share the `SHA256SUMS` and `SHA256SUMS.asc` files with the team for verification and signing. -14. Append the signatures received from the team to the `SHA256SUMS.asc` file. Verify the file using `gpg --verify SHA256SUMS.asc`. Then re-upload the file. +12. Upload the reproducible builds along with `SHA256SUMS-v` and `SHA256SUMS-v.asc` files from the release folder to the newly drafted release. +13. Share the `SHA256SUMS-v` and `SHA256SUMS-v.asc` files with the team for verification and signing. +14. Append the signatures received from the team to the `SHA256SUMS-v.asc` file. Verify the file using `gpg --verify SHA256SUMS-v.asc`. Then re-upload the file. 15. Finalize and publish the release (change it from draft to public). 16. Ensure that the GitHub Actions for `Publish Python 🐍 distributions 📦 to PyPI and TestPyPI` and `Build and push multi-platform docker images` are functioning correctly. Check that the `PyPI` modules published on `https://pypi.org/project/pyln-*` and that the Docker image has been uploaded to Docker Hub. 17. Announce the hotfix release in the core-lightning release-chat channel on Discord and on [BuildOnL2](https://community.corelightning.org/c/general-questions/). diff --git a/plugins/rest-plugin/Cargo.toml b/plugins/rest-plugin/Cargo.toml index 32b74a4f0b3d..09b38fae837f 100644 --- a/plugins/rest-plugin/Cargo.toml +++ b/plugins/rest-plugin/Cargo.toml @@ -26,12 +26,12 @@ rcgen = "0.13" hyper = "1" tower= "0.5" tower-http = { version = "0.6", features = ["cors", "set-header"] } -utoipa = { version = "5", features = ['axum_extras'] } +utoipa = { git = "https://github.com/rustyrussell/utoipa.git", features = ['axum_extras'] } log-panics = "2" socketioxide = "0.15" cln-plugin = { version = "0.5", path = "../../plugins" } cln-rpc = { version = "0.5", path = "../../cln-rpc" } -utoipa-swagger-ui = { version = "9.0.0", features = ["vendored", "axum"] } +utoipa-swagger-ui = { git = "https://github.com/rustyrussell/utoipa.git", features = ["vendored", "axum"] } diff --git a/tools/build-release.sh b/tools/build-release.sh index fb6478903039..ac0d3239e62f 100755 --- a/tools/build-release.sh +++ b/tools/build-release.sh @@ -8,6 +8,7 @@ if [ "$1" = "--inside-docker" ]; then PLTFM="$3" PLTFMVER="$4" ARCH="$5" + MAKEPAR="$6" git config --global --add safe.directory /src/.git git clone /src /build cd /build || exit @@ -15,8 +16,8 @@ if [ "$1" = "--inside-docker" ]; then uv export --format requirements.txt > /tmp/requirements.txt uv pip install -r /tmp/requirements.txt ./configure - uv run make VERSION="$VER" - uv run make install DESTDIR=/"$VER-$PLTFM-$PLTFMVER-$ARCH" RUST_PROFILE=release + uv run make -j"$MAKEPAR" VERSION="$VER" + uv run make -j"$MAKEPAR" install DESTDIR=/"$VER-$PLTFM-$PLTFMVER-$ARCH" RUST_PROFILE=release cd /"$VER-$PLTFM-$PLTFMVER-$ARCH" && tar cvfz /release/clightning-"$VER-$PLTFM-$PLTFMVER-$ARCH".tar.gz -- * echo "Inside docker: build finished" exit 0 @@ -25,6 +26,7 @@ fi FORCE_UNCLEAN=false VERIFY_RELEASE=false WITHOUT_ZIP=false +SUDO= ALL_TARGETS="bin-Fedora bin-Ubuntu docker sign" # ALL_TARGETS="bin-Fedora bin-Ubuntu tarball deb docker sign" @@ -46,6 +48,9 @@ for arg; do --without-zip) WITHOUT_ZIP=true ;; + --sudo) + SUDO=sudo + ;; --help) echo "Usage: [--force-version=] [--force-unclean] [--force-mtime=YYYY-MM-DD] [--verify] [TARGETS]" echo Known targets: "$ALL_TARGETS" @@ -82,6 +87,15 @@ if [ "$VERSION" = "" ]; then exit 1 fi +# Don't forget the v prefix! +case "$VERSION" in + v*) ;; + *) + echo "Version must begin with v! Not $VERSION" >&2 + exit 1 + ;; +esac + # `status --porcelain -u no` suppressed modified! Bug reported... if [ "$(git diff --name-only)" != "" ] && ! $FORCE_UNCLEAN; then echo "Not a clean git directory" >&2 @@ -97,6 +111,9 @@ if [ -z "$MTIME" ]; then exit 1 fi +MAKEPAR=${MAKEPAR:-$(nproc)} +echo "Parallel: $MAKEPAR" + if [ "$VERIFY_RELEASE" = "true" ]; then if [ -f "SHA256SUMS-$VERSION.asc" ] && [ -f "SHA256SUMS-$VERSION" ]; then ALL_TARGETS="bin-Ubuntu" @@ -127,6 +144,8 @@ if [ "$WITHOUT_ZIP" = "false" ]; then # submodcheck needs to know if we have lowdown touch config.vars ./configure --reconfigure + # If you don't have lowdown, your zip file will include it: we assume everyone has it now! + grep -q "HAVE_LOWDOWN=1" config.vars || (echo "Please install lowdown" >&2; exit 1) # If it's a completely clean directory, we need submodules! make submodcheck @@ -163,20 +182,20 @@ for target in $TARGETS; do DOCKERFILE=contrib/docker/Dockerfile.builder.fedora FEDORA_VERSION=$(grep -oP '^FROM fedora:\K[0-9]+' "$DOCKERFILE") docker build -f $DOCKERFILE -t $TAG --load . - docker run --rm=true -v "$(pwd)":/src:ro -v "$RELEASEDIR":/release $TAG /src/tools/build-release.sh --inside-docker "$VERSION" "$platform" "$FEDORA_VERSION" "$ARCH" + docker run --rm=true -v "$(pwd)":/src:ro -v "$RELEASEDIR":/release $TAG /src/tools/build-release.sh --inside-docker "$VERSION" "$platform" "$FEDORA_VERSION" "$ARCH" "$MAKEPAR" docker run --rm=true -w /build $TAG rm -rf /"$VERSION-$platform-$FEDORA_VERSION-$ARCH" /build echo "Fedora Image Built" ;; Ubuntu*) distributions=${platform#Ubuntu-} [ "$distributions" = "Ubuntu" ] && distributions="focal jammy noble" - for d in $distributions; do + for d in $distributions; do # Capitalize the first letter of distro D=$(echo "$d" | awk '{print toupper(substr($0,1,1))substr($0,2)}') - echo "Building Ubuntu $D Image" - docker run --rm -v "$(pwd)":/repo -e FORCE_MTIME="$MTIME" -e FORCE_VERSION="$VERSION" cl-repro-"$d" + echo "Building Ubuntu $D Image" + docker run --rm -v "$(pwd)":/repo -e FORCE_MTIME="$MTIME" -e FORCE_VERSION="$VERSION" -e MAKEPAR="$MAKEPAR" cl-repro-"$d" echo "Ubuntu $D Image Built" - done + done ;; *) echo "No Dockerfile for $platform" >&2 @@ -195,21 +214,21 @@ if [ -z "${TARGETS##* docker *}" ]; then DOCKER_OPTS="$DOCKER_OPTS -t $DOCKER_USER/lightningd:latest" DOCKER_OPTS="$DOCKER_OPTS --cache-to=type=local,dest=/tmp/docker-cache --cache-from=type=local,src=/tmp/docker-cache" echo "Docker Options: $DOCKER_OPTS" - if sudo docker buildx ls | grep -q 'cln-builder'; then - sudo docker buildx use cln-builder + if $SUDO docker buildx ls | grep -q 'cln-builder'; then + $SUDO docker buildx use cln-builder else - sudo docker buildx create --name=cln-builder --use + $SUDO docker buildx create --name=cln-builder --use fi # shellcheck disable=SC2086 - sudo docker buildx build $DOCKER_OPTS . + $SUDO docker buildx build $DOCKER_OPTS . echo "Pushed multi-platform images tagged as $VERSION and latest" fi if [ -z "${TARGETS##* sign *}" ]; then echo "Signing Release" cd release/ || exit - sha256sum clightning-"$VERSION"* > SHA256SUMS - gpg -sb --armor -o SHA256SUMS.asc"$(gpgconf --list-options gpg | awk -F: '$1 == "default-key" {print $10}' | tr -d '"')" SHA256SUMS + sha256sum clightning-"$VERSION"* > SHA256SUMS-"$VERSION" + gpg -sb --armor -o SHA256SUMS-"$VERSION".asc "$(gpgconf --list-options gpg | awk -F: '$1 == "default-key" {print $10}' | tr -d '"')" SHA256SUMS-"$VERSION" cd .. echo "Release Signed" fi diff --git a/tools/repro-build.sh b/tools/repro-build.sh index b5a3af611507..c7b6e17f163e 100755 --- a/tools/repro-build.sh +++ b/tools/repro-build.sh @@ -9,19 +9,19 @@ export LANG LC_ALL for arg; do case "$arg" in --force-version=*) - FORCE_VERSION=${arg#*=} + FORCE_VERSION=${arg#*=} ;; - --force-mtime=*) - FORCE_MTIME=${arg#*=} - ;; - --help) - echo "Usage: [--force-version=] [--force-mtime=YYYY-MM-DD]" - exit 0 - ;; - *) - echo "Unknown arg $arg" >&2 - exit 1 - ;; + --force-mtime=*) + FORCE_MTIME=${arg#*=} + ;; + --help) + echo "Usage: [--force-version=] [--force-mtime=YYYY-MM-DD]" + exit 0 + ;; + *) + echo "Unknown arg $arg" >&2 + exit 1 + ;; esac shift done @@ -56,6 +56,7 @@ fi ARCH=$(dpkg --print-architecture) PLATFORM="$OS"-"$VER" VERSION=${FORCE_VERSION:-$(git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's,.*/clightning-\(v[0-9.rc\-]*\)$,\1,p')} +MAKEPAR=${MAKEPAR:-1} # eg. ## [0.6.3] - 2019-01-09: "The Smallblock Conspiracy" # Skip 'v' here in $VERSION @@ -160,8 +161,8 @@ $INST $(cut -c66- < /tmp/SHASUMS) # Once everyone has gcc8, we can use CC="gcc -ffile-prefix-map=$(pwd)=/home/clightning" ./configure --prefix=/usr CC="gcc -fdebug-prefix-map=$(pwd)=/home/clightning" # libwally wants "python". Seems to work to force it here. -make PYTHON_VERSION=3 VERSION="$VERSION" -make install DESTDIR=inst/ +make -j"$MAKEPAR" PYTHON_VERSION=3 VERSION="$VERSION" +make -j"$MAKEPAR" install DESTDIR=inst/ cd inst && tar --sort=name \ --mtime="$MTIME 00:00Z" \