Merge commit 'ae706ad83f32bdb999c829b28e8645fbbae0514b' into catchup/long_lived_datalayer_merkle_blob_from_main_ae706ad83f32bdb999c829b28e8645fbbae0514b

Author: altendky

.github/workflows/build-linux-installer-deb.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ${{ matrix.os.runs-on }}
     needs:
       - version
-    container: chianetwork/ubuntu-20.04-builder:latest
+    container: chianetwork/ubuntu-22.04-builder:latest
     timeout-minutes: 120
     strategy:
       fail-fast: false
@@ -257,18 +257,10 @@ jobs:
               arm: [linux, arm64]
               intel: [ubuntu-latest]
         distribution:
-          - name: debian:bullseye
-            type: debian
-            # https://packages.debian.org/bullseye/python/python3 (3.9)
-            url: "docker://debian:bullseye"
           - name: debian:bookworm
             type: debian
             # https://packages.debian.org/bookworm/python/python3 (3.11)
             url: "docker://debian:bookworm"
-          - name: ubuntu:focal (20.04)
-            type: ubuntu
-            # https://packages.ubuntu.com/focal/python3 (20.04, 3.8)
-            url: "docker://ubuntu:focal"
           - name: ubuntu:jammy (22.04)
             type: ubuntu
             # https://packages.ubuntu.com/jammy/python3 (22.04, 3.10)

.github/workflows/test-install-scripts.yml

@@ -153,10 +153,6 @@ jobs:
           - name: arch:latest
             type: arch
             url: "docker://archlinux:latest"
-          - name: debian:bullseye
-            type: debian
-            # https://packages.debian.org/bullseye/python/python3 (3.9)
-            url: "docker://debian:bullseye"
           - name: debian:bookworm
             type: debian
             # https://packages.debian.org/bookworm/python/python3 (3.11)
@@ -175,10 +171,6 @@ jobs:
           - name: rockylinux:9
             type: rocky
             url: "docker://rockylinux:9"
-          - name: ubuntu:focal (20.04)
-            type: ubuntu
-            # https://packages.ubuntu.com/focal/python3 (20.04, 3.8)
-            url: "docker://ubuntu:focal"
           - name: ubuntu:jammy (22.04)
             type: ubuntu
             # https://packages.ubuntu.com/jammy/python3 (22.04, 3.10)
@@ -243,7 +235,7 @@ jobs:
           apt-get --yes update
           apt-get install --yes git lsb-release sudo
           MINIMUM=3.9
-          if ! apt satisfy --yes "python3-venv (>= ${MINIMUM})"
+          if ! apt-get satisfy --yes "python3-venv (>= ${MINIMUM})"
           then
             apt-get install --yes python${MINIMUM}-venv
           fi

build_scripts/build_linux_deb-2-installer.sh

@@ -90,7 +90,7 @@ if [ "$PLATFORM" = "arm64" ]; then
   # This is a temporary fix.
   # https://github.com/jordansissel/fpm/issues/1801#issuecomment-919877499
   # @TODO Consolidates the process to amd64 if the issue of electron-builder is resolved
-  sudo apt -y install ruby ruby-dev
+  sudo apt-get -y install ruby ruby-dev
   # ERROR:  Error installing fpm:
   #     The last version of dotenv (>= 0) to support your Ruby & RubyGems was 2.8.1. Try installing it with `gem install dotenv -v 2.8.1` and then running the current command again
   #     dotenv requires Ruby version >= 3.0. The current ruby version is 2.7.0.0.

chia/consensus/blockchain.py

@@ -613,20 +613,13 @@ async def _reconsider_peak(
             [fork_add.coin for fork_add in fork_info.additions_since_fork.values() if fork_add.is_coinbase],
         )
 
-    def get_next_difficulty(self, header_hash: bytes32, new_slot: bool) -> uint64:
+    def get_next_sub_slot_iters_and_difficulty(self, header_hash: bytes32, new_slot: bool) -> tuple[uint64, uint64]:
         curr = self.try_block_record(header_hash)
         assert curr is not None
         if curr.height <= 2:
-            return self.constants.DIFFICULTY_STARTING
+            return self.constants.SUB_SLOT_ITERS_STARTING, self.constants.DIFFICULTY_STARTING
 
-        return get_next_sub_slot_iters_and_difficulty(self.constants, new_slot, curr, self)[1]
-
-    def get_next_slot_iters(self, header_hash: bytes32, new_slot: bool) -> uint64:
-        curr = self.try_block_record(header_hash)
-        assert curr is not None
-        if curr.height <= 2:
-            return self.constants.SUB_SLOT_ITERS_STARTING
-        return get_next_sub_slot_iters_and_difficulty(self.constants, new_slot, curr, self)[0]
+        return get_next_sub_slot_iters_and_difficulty(self.constants, new_slot, curr, self)
 
     async def get_sp_and_ip_sub_slots(
         self, header_hash: bytes32

chia/full_node/full_node.py

@@ -102,6 +102,7 @@ class PeakPostProcessingResult:
     fns_peak_result: FullNodeStorePeakResult  # The result of calling FullNodeStore.new_peak
     hints: list[tuple[bytes32, bytes]]  # The hints added to the DB
     lookup_coin_ids: list[bytes32]  # The coin IDs that we need to look up to notify wallets of changes
+    signage_points: list[tuple[RespondSignagePoint, WSChiaConnection, Optional[EndOfSubSlotBundle]]]
 
 
 @dataclasses.dataclass(frozen=True)
@@ -837,7 +838,7 @@ async def send_peak_to_timelords(
             peak_block = await self.blockchain.get_full_peak()
         if peak_block is not None:
             peak = self.blockchain.block_record(peak_block.header_hash)
-            difficulty = self.blockchain.get_next_difficulty(peak.header_hash, False)
+            difficulty = self.blockchain.get_next_sub_slot_iters_and_difficulty(peak.header_hash, False)[1]
             ses: Optional[SubEpochSummary] = next_sub_epoch_summary(
                 self.constants,
                 self.blockchain,
@@ -1807,10 +1808,9 @@ async def signage_point_post_processing(
             # Makes sure to potentially update the difficulty if we are past the peak (into a new sub-slot)
             assert ip_sub_slot is not None
             if request.challenge_chain_vdf.challenge != ip_sub_slot.challenge_chain.get_hash():
-                next_difficulty = self.blockchain.get_next_difficulty(peak.header_hash, True)
-                next_sub_slot_iters = self.blockchain.get_next_slot_iters(peak.header_hash, True)
-                difficulty = next_difficulty
-                sub_slot_iters = next_sub_slot_iters
+                sub_slot_iters, difficulty = self.blockchain.get_next_sub_slot_iters_and_difficulty(
+                    peak.header_hash, True
+                )
         else:
             difficulty = self.constants.DIFFICULTY_STARTING
             sub_slot_iters = self.constants.SUB_SLOT_ITERS_STARTING
@@ -1845,8 +1845,7 @@ async def peak_post_processing(
         """
 
         record = state_change_summary.peak
-        difficulty = self.blockchain.get_next_difficulty(record.header_hash, False)
-        sub_slot_iters = self.blockchain.get_next_slot_iters(record.header_hash, False)
+        sub_slot_iters, difficulty = self.blockchain.get_next_sub_slot_iters_and_difficulty(record.header_hash, False)
 
         self.log.info(
             f"🌱 Updated peak to height {record.height}, weight {record.weight}, "
@@ -1895,6 +1894,7 @@ async def peak_post_processing(
             difficulty,
         )
 
+        signage_points: list[tuple[RespondSignagePoint, WSChiaConnection, Optional[EndOfSubSlotBundle]]] = []
         if fns_peak_result.new_signage_points is not None and peer is not None:
             for index, sp in fns_peak_result.new_signage_points:
                 assert (
@@ -1903,8 +1903,13 @@ async def peak_post_processing(
                     and sp.rc_vdf is not None
                     and sp.rc_proof is not None
                 )
-                await self.signage_point_post_processing(
-                    RespondSignagePoint(index, sp.cc_vdf, sp.cc_proof, sp.rc_vdf, sp.rc_proof), peer, sub_slots[1]
+                # Collect the data for networking outside the mutex
+                signage_points.append(
+                    (
+                        RespondSignagePoint(index, sp.cc_vdf, sp.cc_proof, sp.rc_vdf, sp.rc_proof),
+                        peer,
+                        sub_slots[1],
+                    )
                 )
 
         if sub_slots[1] is None:
@@ -1934,6 +1939,7 @@ async def peak_post_processing(
             fns_peak_result,
             hints_to_add,
             lookup_coin_ids,
+            signage_points=signage_points,
         )
 
     async def peak_post_processing_2(
@@ -1948,6 +1954,8 @@ async def peak_post_processing_2(
         with peers
         """
         record = state_change_summary.peak
+        for signage_point in ppp_result.signage_points:
+            await self.signage_point_post_processing(*signage_point)
         for new_peak_item in ppp_result.mempool_peak_result:
             self.log.debug(f"Added transaction to mempool: {new_peak_item.transaction_id}")
             mempool_item = self.mempool_manager.get_mempool_item(new_peak_item.transaction_id)
@@ -2673,8 +2681,9 @@ async def add_end_of_sub_slot(
 
             peak = self.blockchain.get_peak()
             if peak is not None and peak.height > 2:
-                next_sub_slot_iters = self.blockchain.get_next_slot_iters(peak.header_hash, True)
-                next_difficulty = self.blockchain.get_next_difficulty(peak.header_hash, True)
+                next_sub_slot_iters, next_difficulty = self.blockchain.get_next_sub_slot_iters_and_difficulty(
+                    peak.header_hash, True
+                )
             else:
                 next_sub_slot_iters = self.constants.SUB_SLOT_ITERS_STARTING
                 next_difficulty = self.constants.DIFFICULTY_STARTING

chia/full_node/full_node_api.py

@@ -726,7 +726,9 @@ async def respond_signage_point(
                 return None
             peak = self.full_node.blockchain.get_peak()
             if peak is not None and peak.height > self.full_node.constants.MAX_SUB_SLOT_BLOCKS:
-                next_sub_slot_iters = self.full_node.blockchain.get_next_slot_iters(peak.header_hash, True)
+                next_sub_slot_iters = self.full_node.blockchain.get_next_sub_slot_iters_and_difficulty(
+                    peak.header_hash, True
+                )[0]
                 sub_slots_for_peak = await self.full_node.blockchain.get_sp_and_ip_sub_slots(peak.header_hash)
                 assert sub_slots_for_peak is not None
                 ip_sub_slot: Optional[EndOfSubSlotBundle] = sub_slots_for_peak[1]

install-timelord.sh

@@ -59,22 +59,6 @@ echo "${CHIAVDF_POETRY_INFO_VERSION}"
 CHIAVDF_VERSION="chiavdf==${CHIAVDF_POETRY_INFO_VERSION}"
 echo "${CHIAVDF_VERSION}"
 
-ubuntu_cmake_install() {
-  UBUNTU_PRE_2004=$(python -c 'import subprocess; id = subprocess.run(["lsb_release", "-is"], stdout=subprocess.PIPE); version = subprocess.run(["lsb_release", "-rs"], stdout=subprocess.PIPE); print(id.stdout.decode("ascii") == "Ubuntu\n" and float(version.stdout) < float(20.04))')
-  if [ "$UBUNTU_PRE_2004" = "True" ]; then
-    echo "Installing CMake with snap."
-    sudo apt-get install snapd -y
-    sudo apt-get remove --purge cmake -y
-    hash -r
-    sudo snap install cmake --classic
-    # shellcheck disable=SC1091
-    . /etc/profile
-  else
-    echo "Ubuntu 20.04LTS and newer support CMake 3.16+"
-    sudo apt-get install cmake -y
-  fi
-}
-
 symlink_vdf_bench() {
   if [ ! -e vdf_bench ] && [ -e venv/lib/"$1"/site-packages/vdf_bench ]; then
     echo ln -s venv/lib/"$1"/site-packages/vdf_bench
@@ -111,8 +95,6 @@ if [ -e "$THE_PATH" ]; then
 else
   if [ -e venv/bin/python ] && test "$UBUNTU_DEBIAN"; then
     echo "Installing chiavdf dependencies on Ubuntu/Debian"
-    # If Ubuntu version is older than 20.04LTS then upgrade CMake
-    ubuntu_cmake_install
     # Install remaining needed development tools - assumes venv and prior run of install.sh
     echo "apt-get install libgmp-dev libboost-python-dev $PYTHON_DEV_DEPENDENCY libboost-system-dev build-essential -y"
     sudo apt-get install libgmp-dev libboost-python-dev "$PYTHON_DEV_DEPENDENCY" libboost-system-dev build-essential -y

poetry.lock

@@ -832,7 +832,7 @@ typing-extensions = "*"
 type = "git"
 url = "https://github.com/chia-network/chia_rs"
 reference = "long_lived/initial_datalayer"
-resolved_reference = "151269e9f85bd2c87640dab05c9778e20ee19a56"
+resolved_reference = "92c7c0e7042efc6bbe597829cc90e86d5ba33e7d"
 subdirectory = "wheel/"
 
 [[package]]
@@ -2649,15 +2649,15 @@ wheel = "*"
 
 [[package]]
 name = "pytest-xdist"
-version = "3.6.1"
+version = "3.7.0"
 description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs"
 optional = true
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 groups = ["main"]
 markers = "extra == \"dev\""
 files = [
-    {file = "pytest_xdist-3.6.1-py3-none-any.whl", hash = "sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7"},
-    {file = "pytest_xdist-3.6.1.tar.gz", hash = "sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d"},
+    {file = "pytest_xdist-3.7.0-py3-none-any.whl", hash = "sha256:7d3fbd255998265052435eb9daa4e99b62e6fb9cfb6efd1f858d4d8c0c7f0ca0"},
+    {file = "pytest_xdist-3.7.0.tar.gz", hash = "sha256:f9248c99a7c15b7d2f90715df93610353a485827bc06eefb6566d23f6400f126"},
 ]
 
 [package.dependencies]
@@ -3506,4 +3506,4 @@ upnp = ["miniupnpc"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.9, <4"
-content-hash = "dc2647eb50337c908de6d510644356ca0ba1d289e782e527b6525d87e2edec9d"
+content-hash = "6a5b2af260a6b53a529c7e283f456b910aef9b332995a10436456399181d2937"

pyproject.toml

@@ -1,16 +1,22 @@
-[tool.poetry]
+[project]
 name = "chia-blockchain"
-# see [tool.poetry-dynamic-versioning]
-version = "0.0.0"
+dynamic = [ "dependencies", "version" ]
 description = "Chia blockchain full node, farmer, timelord, and wallet."
-authors = ["Mariano Sorgente <[email protected]>"]
-license = "Apache-2.0"
-readme = "README.md"
+requires-python = ">=3.9, <4"
+authors = [
+    {name="Mariano Sorgente", email="[email protected]"}
+]
+license = {text="Apache-2.0"}
+readme = {file = "README.md", content-type = "text/markdown"}
 keywords= ["chia", "blockchain", "node"]
 homepage = "https://chia.net/"
+
+[tool.poetry]
+# see [tool.poetry-dynamic-versioning]
 packages = [{ include = "chia"}, { include = "mozilla-ca/cacert.pem" }]
+version = "0.0.0"
 
-[tool.poetry.scripts]
+[project.scripts]
 chia = "chia.cmds.chia:main"
 chia_daemon = "chia.daemon.server:main"
 chia_wallet = "chia.server.start_wallet:main"
@@ -32,12 +38,11 @@ name = "chia"
 url = "https://pypi.chia.net/simple/"
 priority = "supplemental"
 
-[tool.poetry.urls]
-"Source" = "https://github.com/Chia-Network/chia-blockchain/"
-"Changelog" = "https://github.com/Chia-Network/chia-blockchain/blob/main/CHANGELOG.md"
+[project.urls]
+source = "https://github.com/Chia-Network/chia-blockchain/"
+changelog = "https://github.com/Chia-Network/chia-blockchain/blob/main/CHANGELOG.md"
 
 [tool.poetry.dependencies]
-python = ">=3.9, <4"
 aiofiles = ">=24.1.0"  # Async IO for files
 aiohttp = ">=3.10.4"  # HTTP server for full node rpc
 aiosqlite = ">=0.20.0"  # asyncio wrapper for sqlite, to store blocks
@@ -140,10 +145,3 @@ bump = true
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"]
 build-backend = "poetry_dynamic_versioning.backend"
-
-[project]
-# duplicating until poetry v2, then revisit https://github.com/Chia-Network/chia-blockchain/pull/18797
-name = "chia-blockchain"
-# This has to match the poetry python entry above
-requires-python = ">=3.9, <4"
-dynamic = ["version"]