Merge branch 'main' into electra-support

Author: rodrigo-o

.github/config/assertoor/cl-stability-check.yml

@@ -34,7 +34,7 @@ tasks:
       name: check_consensus_block_proposals
       title: "Wait for block proposal from ${validatorPairName}"
       config:
-        minTransactionCount: 240
+        minTransactionCount: 80 # For some reason the tx fuzz is working different than the old spammer, we need to check it
       configVars:
         validatorNamePattern: "validatorPairName"
 

Makefile

@@ -105,6 +105,10 @@ kurtosis.connect:
 kurtosis.connect.iex:
 	iex --sname client --remsh lambdaconsensus --cookie $(KURTOSIS_COOKIE)
 
+#💻 kurtosis.assertoor: @ Execute the assertoor network params in .github
+kurtosis.assertoor: kurtosis.clean kurtosis.setup.lambdaconsensus
+	kurtosis run --enclave $(KURTOSIS_ENCLAVE) $(KURTOSIS_DIR) --args-file .github/config/assertoor/network-params.yml
+
 #💻 nix: @ Start a nix environment.
 nix:
 	nix develop

README.md

@@ -109,6 +109,14 @@ You can specify a URL to fetch it from with the "--checkpoint-sync-url" flag:
 iex -S mix run -- --checkpoint-sync-url <your_url_here>
 ```
 
+or you can specify mulitple urls by passing a comma separated list of urls:
+
+```shell
+iex -S mix run -- --checkpoint-sync-url "<url1>, <url2>, ..."
+```
+
+If multiple urls are provided the downloaded state will be compared for all urls and fail if even one of them differs from the rest
+
 Some public endpoints can be found in [eth-clients.github.io/checkpoint-sync-endpoints](https://eth-clients.github.io/checkpoint-sync-endpoints/).
 
 > [!IMPORTANT]

config/runtime.exs

@@ -103,7 +103,13 @@ config :lambda_ethereum_consensus, LambdaEthereumConsensus.Store.Db, dir: datadi
 # We use put_env here as we need this immediately after to read the state.
 Application.put_env(:lambda_ethereum_consensus, ChainSpec, config: chain_config)
 
-strategy = StoreSetup.make_strategy!(testnet_dir, checkpoint_sync_url)
+checkpoint_urls =
+  case checkpoint_sync_url do
+    urls when is_binary(urls) -> urls |> String.split(",") |> Enum.map(&String.trim/1)
+    nil -> nil
+  end
+
+strategy = StoreSetup.make_strategy!(testnet_dir, checkpoint_urls)
 
 genesis_validators_root =
   case strategy do

lib/beacon_api/controllers/v1/config_controller.ex

@@ -17,12 +17,6 @@ defmodule BeaconApi.V1.ConfigController do
   ]
   @chain_spec_hex_fields [
     "TERMINAL_BLOCK_HASH",
-    "GENESIS_FORK_VERSION",
-    "ALTAIR_FORK_VERSION",
-    "BELLATRIX_FORK_VERSION",
-    "CAPELLA_FORK_VERSION",
-    "DENEB_FORK_VERSION",
-    "ELECTRA_FORK_VERSION",
     "DEPOSIT_CONTRACT_ADDRESS",
     "MESSAGE_DOMAIN_INVALID_SNAPPY",
     "MESSAGE_DOMAIN_VALID_SNAPPY"
@@ -43,9 +37,17 @@ defmodule BeaconApi.V1.ConfigController do
     |> Map.drop(@chain_spec_removed_keys)
     |> rename_keys(@chain_spec_renamed_keys)
     |> Map.new(fn
-      {k, v} when is_integer(v) -> {k, Integer.to_string(v)}
-      {k, v} when k in @chain_spec_hex_fields -> {k, Utils.hex_encode(v)}
-      {k, v} -> {k, v}
+      {k, v} when is_integer(v) ->
+        {k, Integer.to_string(v)}
+
+      {k, v} when k in @chain_spec_hex_fields ->
+        {k, Utils.hex_encode(v)}
+
+      {k, v} when is_binary(v) ->
+        if String.ends_with?(k, "_FORK_VERSION"), do: {k, Utils.hex_encode(v)}, else: {k, v}
+
+      {k, v} ->
+        {k, v}
     end)
   end
 

lib/lambda_ethereum_consensus/beacon/store_setup.ex

@@ -20,15 +20,15 @@ defmodule LambdaEthereumConsensus.Beacon.StoreSetup do
   @doc """
   Args: at least one can be nil.
   - testnet_dir: directory of a testnet configuration, including ssz and yaml config.
-  - checkpoint_sync_url: a url where checkpoint sync can be performed.
+  - checkpoint_sync_url: list of urls where checkpoint sync can be performed.
 
   Return value: a store setup strategy, which is one of the following:
   - {:file, anchor_state}: path of an ssz file to get the genesis state from.
-  - {:checkpoint_sync_url, url}: url to get the genesis state from if performing checkpoint sync.
+  - {:checkpoint_sync_url, url}: list of urls to get the genesis state from if performing checkpoint sync.
   - :db : the genesis state and store can only be recovered from the db.
   """
   def make_strategy!(nil, nil), do: :db
-  def make_strategy!(nil, url) when is_binary(url), do: {:checkpoint_sync_url, url}
+  def make_strategy!(nil, urls) when is_list(urls), do: {:checkpoint_sync_url, urls}
 
   def make_strategy!(dir, nil) when is_binary(dir) do
     Path.join(dir, "genesis.ssz")
@@ -55,14 +55,14 @@ defmodule LambdaEthereumConsensus.Beacon.StoreSetup do
     store
   end
 
-  def setup!({:checkpoint_sync_url, checkpoint_url}) do
+  def setup!({:checkpoint_sync_url, checkpoint_urls}) do
     case restore_state_from_db() do
       {:ok, store} ->
         Logger.warning("[Checkpoint sync] Recent state found. Ignoring the checkpoint URL.")
         store
 
       _ ->
-        fetch_state_from_url(checkpoint_url)
+        fetch_and_compare_state_from_urls(checkpoint_urls)
     end
   end
 
@@ -87,8 +87,12 @@ defmodule LambdaEthereumConsensus.Beacon.StoreSetup do
   @spec get_deposit_snapshot!() :: DepositTreeSnapshot.t() | nil
   def get_deposit_snapshot!(), do: get_deposit_snapshot!(get_strategy!())
 
+  # The endpoint for deposit snapshots is deprecated in electra and will be removed in Fulu
+  # https://github.com/ethereum/beacon-APIs/pull/494
+  # For this reason we don't compare the deposits from the urls as most checkpoints are returning error 500
   @spec get_deposit_snapshot!(store_setup_strategy()) :: DepositTreeSnapshot.t() | nil
-  def get_deposit_snapshot!({:checkpoint_sync_url, url}), do: fetch_deposit_snapshot(url)
+  def get_deposit_snapshot!({:checkpoint_sync_url, urls}),
+    do: fetch_deposit_snapshot(List.first(urls))
 
   def get_deposit_snapshot!(:db) do
     case StoreDb.fetch_deposits_snapshot() do
@@ -129,28 +133,65 @@ defmodule LambdaEthereumConsensus.Beacon.StoreSetup do
     end
   end
 
-  defp fetch_state_from_url(url) do
+  defp fetch_and_compare_state_from_urls(urls) do
     Logger.info("[Checkpoint sync] Initiating checkpoint sync")
 
+    # Fetch last finalized block for all urls
+    blocks = for {:ok, res} <- Enum.map(urls, &CheckpointSync.get_block/1), do: res
+
+    case Enum.uniq(blocks) do
+      [_] ->
+        Logger.info(
+          "[Checkpoin sync] Received the same last finalized block from #{length(blocks)} checkpoint nodes"
+        )
+
+      _ ->
+        Logger.error(
+          "[Checkpoint sync] Received inconsistent last finalized block from #{length(blocks)} checkpoint nodes"
+        )
+
+        Logger.flush()
+        System.halt(1)
+    end
+
     genesis_validators_root = ChainSpec.get_genesis_validators_root()
 
+    # All urls returned the same last finalized block, we will trust the first to get the state
+    {anchor_state, anchor_block} = fetch_state_from_url(genesis_validators_root, List.first(urls))
+
+    first_block = List.first(blocks)
+
+    if anchor_state.latest_block_header.parent_root == first_block.message.parent_root do
+      {:ok, store} = Store.get_forkchoice_store(anchor_state, anchor_block)
+
+      # Save store in DB
+      StoreDb.persist_store(store)
+
+      store
+    else
+      Logger.error(
+        "[Checkpoint sync] Root mismatch when comparing latest finalized block with downloaded state"
+      )
+
+      Logger.flush()
+      System.halt(1)
+    end
+  end
+
+  defp fetch_state_from_url(genesis_validators_root, url) do
     case CheckpointSync.get_finalized_block_and_state(url, genesis_validators_root) do
       {:ok, {anchor_state, anchor_block}} ->
         Logger.info(
-          "[Checkpoint sync] Received beacon state and block",
+          "[Checkpoint sync] Received beacon state and block from URL #{url}",
           slot: anchor_state.slot
         )
 
-        # We already checked block and state match
-        {:ok, store} = Store.get_forkchoice_store(anchor_state, anchor_block)
-
-        # Save store in DB
-        StoreDb.persist_store(store)
-
-        store
+        {anchor_state, anchor_block}
 
       _ ->
-        Logger.error("[Checkpoint sync] Failed to fetch the latest finalized state and block")
+        Logger.error(
+          "[Checkpoint sync] Failed to fetch the latest finalized state and block for URL: #{url}"
+        )
 
         Logger.flush()
         System.halt(1)

lib/lambda_ethereum_consensus/p2p/gossip/blob_sidecar.ex

@@ -19,7 +19,7 @@ defmodule LambdaEthereumConsensus.P2P.Gossip.BlobSideCar do
       Logger.debug("[Gossip] Blob sidecar received, with index #{blob_index}")
       Libp2pPort.validate_message(msg_id, :accept)
       # TODO: (#1406) Enhance the API to reduce unnecessary wrappers (:ok + list)
-      PendingBlocks.process_blobs(store, {:ok, [blob]})
+      PendingBlocks.process_blobs(store, {:ok, [blob]}) |> then(&elem(&1, 1))
     else
       {:error, reason} ->
         Logger.warning("[Gossip] Blob rejected, reason: #{inspect(reason)}")

mix.exs

@@ -61,7 +61,7 @@ defmodule LambdaEthereumConsensus.MixProject do
       {:open_api_spex, "~> 3.18"},
       {:crc32c, git: "https://github.com/lambdaclass/crc32c", branch: "bump-rustler-32"},
       {:recode, "~> 0.7", only: [:dev, :test]},
-      {:sentry, "~> 10.8.0"},
+      {:sentry, "~> 10.9.0"},
       {:prom_ex, "~> 1.11.0"},
       {:flama, git: "https://github.com/lambdaclass/ht1223_tracer"},
       {:uuid, "~> 1.1"},

mix.lock

@@ -1,16 +1,16 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"},
   "aja": {:hex, :aja, "0.7.4", "5f14885df9fdd71e17cfc5f51ec42fe4b8328b23dd7ac34aa4acae6b77843819", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "e7d295357288558f808431f8cc3ce313b3b5d02b20a446433da48490440c7286"},
-  "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
+  "benchee": {:hex, :benchee, "1.4.0", "9f1f96a30ac80bab94faad644b39a9031d5632e517416a8ab0a6b0ac4df124ce", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "299cd10dd8ce51c9ea3ddb74bb150f93d25e968f93e4c1fa31698a8e4fa5d715"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.12", "053f0e32700cbec356280c0e835df425a3be4bc1e0627b714330ad9d0f05497f", [:mix], [], "hexpm", "3dca286b2186055ba0c9449b4e95b97bf1b57b47c1f2644555879e659960c224"},
   "certifi": {:hex, :certifi, "2.14.0", "ed3bef654e69cde5e6c022df8070a579a79e8ba2368a00acf3d75b82d9aceeed", [:rebar3], [], "hexpm", "ea59d87ef89da429b8e905264fdec3419f84f2215bb3d81e07a18aac919026c3"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "cowboy": {:hex, :cowboy, "2.13.0", "09d770dd5f6a22cc60c071f432cd7cb87776164527f205c5a6b0f24ff6b38990", [:make, :rebar3], [{:cowlib, ">= 2.14.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "e724d3a70995025d654c1992c7b11dbfea95205c047d86ff9bf1cda92ddc5614"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
-  "cowlib": {:hex, :cowlib, "2.14.0", "623791c56c1cc9df54a71a9c55147a401549917f00a2e48a6ae12b812c586ced", [:make, :rebar3], [], "hexpm", "0af652d1550c8411c3b58eed7a035a7fb088c0b86aff6bc504b0bc3b7f791aa2"},
+  "cowlib": {:hex, :cowlib, "2.15.0", "3c97a318a933962d1c12b96ab7c1d728267d2c523c25a5b57b0f93392b6e9e25", [:make, :rebar3], [], "hexpm", "4f00c879a64b4fe7c8fcb42a4281925e9ffdb928820b03c3ad325a617e857532"},
   "crc32c": {:git, "https://github.com/lambdaclass/crc32c", "457d72862d90b57c7ff079673872c2045b64cac9", [branch: "bump-rustler-32"]},
-  "credo": {:hex, :credo, "1.7.11", "d3e805f7ddf6c9c854fd36f089649d7cf6ba74c42bc3795d587814e3c9847102", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "56826b4306843253a66e47ae45e98e7d284ee1f95d53d1612bb483f88a8cf219"},
+  "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"},
   "eep": {:git, "https://github.com/virtan/eep", "8f6e5e3ade0606390d928830db61350a5451dda8", [branch: "master"]},
@@ -66,7 +66,7 @@
   "rexbug": {:hex, :rexbug, "1.0.6", "024071c67d970151fbdc06f299faf8db3e1b2ac759a28623a9cc80a517fc74f2", [:mix], [{:mix_test_watch, ">= 0.5.0", [hex: :mix_test_watch, repo: "hexpm", optional: true]}, {:redbug, "~> 1.2", [hex: :redbug, repo: "hexpm", optional: false]}], "hexpm", "148ea724979413e9fd84ca3b4bb5d2d8b840ac481adfd645f5846fda409a642c"},
   "rustler": {:hex, :rustler, "0.36.1", "2d4b1ff57ea2789a44756a40dbb5fbb73c6ee0a13d031dcba96d0a5542598a6a", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "f3fba4ad272970e0d1bc62972fc4a99809651e54a125c5242de9bad4574b2d02"},
   "scrypt_elixir": {:hex, :scrypt_elixir_copy, "0.1.1", "2b23573e8d9e6c93c8116cd17f9b453b6ebf0725b5317ecaeacaf73353a4dbd3", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "1eb5768b6b6c657770cbc00a9724f47bad4e9d664a2da3916030d591223561e7"},
-  "sentry": {:hex, :sentry, "10.8.1", "aa45309785e1521416225adb16e0b4d8b957578804527f3c7babb6fefbc5e456", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "495b3cdadad90ba72eef973aa3dec39b3b8b2a362fe87e2f4ef32133ac3b4097"},
+  "sentry": {:hex, :sentry, "10.9.0", "503575bc98ef268ad75e9792e17637ab7b270ed8036614f777a1833272409016", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_options, "~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_ownership, "~> 0.3.0 or ~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_live_view, "~> 0.20 or ~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "9abf07e6a757f6650e2429b5773f546ff119f6980b9bb02067a7eb510a75c9f2"},
   "snappyer": {:hex, :snappyer, "1.2.10", "023e9ae00e969b0997208b5de7d3b12bb46ec6bc5411e8dc53e7b3f435b8f0fd", [:rebar3], [], "hexpm", "f55bd9ed147e7163cb3acd1e431a7ff2c9e31ceacbb8308786094fb64551c284"},
   "sourceror": {:hex, :sourceror, "1.5.0", "3e65d5fbb1a8e2864ad6411262c8018fee73474f5789dda12285c82999253d5d", [:mix], [], "hexpm", "4a32b5d189d8453f73278c15712f8731b89e9211e50726b798214b303b51bfc7"},
   "sse": {:hex, :sse, "0.4.0", "f17affacbc4618bac07590eec7bff849aa27d1f71bb3d41da3fd3cb255d16910", [:mix], [{:event_bus, ">= 1.6.0", [hex: :event_bus, repo: "hexpm", optional: false]}, {:plug, ">= 1.4.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2dfb9923725b9d5292763c3de9b7798713f5771522823e961a250204917d7efb"},