diff --git a/.github/config/assertoor/network-params.yml b/.github/config/assertoor/network-params.yml index ccacbff5b..fc6d0459e 100644 --- a/.github/config/assertoor/network-params.yml +++ b/.github/config/assertoor/network-params.yml @@ -20,15 +20,16 @@ participants: keymanager_enabled: true additional_services: -# - assertoor + - assertoor - tx_spammer + - blob_spammer - dora -#assertoor_params: -# run_stability_check: false -# run_block_proposal_check: false -# tests: [] - # - https://raw.githubusercontent.com/lambdaclass/lambda_ethereum_consensus/refs/heads/main/.github/config/assertoor/cl-stability-check.yml +assertoor_params: + run_stability_check: false + run_block_proposal_check: false + tests: + - https://raw.githubusercontent.com/lambdaclass/lambda_ethereum_consensus/refs/heads/main/.github/config/assertoor/cl-stability-check.yml tx_spammer_params: tx_spammer_extra_args: ["--txcount=3", "--accounts=80"] diff --git a/lib/beacon_api/controllers/error_controller.ex b/lib/beacon_api/controllers/error_controller.ex index 2b9fa950c..f28843c5e 100644 --- a/lib/beacon_api/controllers/error_controller.ex +++ b/lib/beacon_api/controllers/error_controller.ex @@ -1,8 +1,11 @@ defmodule BeaconApi.ErrorController do + require Logger use BeaconApi, :controller @spec bad_request(Plug.Conn.t(), binary()) :: Plug.Conn.t() def bad_request(conn, message) do + Logger.error("Bad request: #{message}, path: #{conn.request_path}") + conn |> put_status(400) |> json(%{ @@ -13,6 +16,8 @@ defmodule BeaconApi.ErrorController do @spec not_found(Plug.Conn.t(), any) :: Plug.Conn.t() def not_found(conn, _params) do + Logger.error("Resource not found, path: #{conn.request_path}") + conn |> put_status(404) |> json(%{ @@ -23,6 +28,8 @@ defmodule BeaconApi.ErrorController do @spec internal_error(Plug.Conn.t(), any) :: Plug.Conn.t() def internal_error(conn, _params) do + Logger.error("Internal server error, path: #{conn.request_path}") + conn |> put_status(500) |> json(%{ diff --git a/lib/beacon_api/controllers/v1/beacon_controller.ex b/lib/beacon_api/controllers/v1/beacon_controller.ex index 1f33036bf..4b1cc3ce2 100644 --- a/lib/beacon_api/controllers/v1/beacon_controller.ex +++ b/lib/beacon_api/controllers/v1/beacon_controller.ex @@ -25,12 +25,15 @@ defmodule BeaconApi.V1.BeaconController do def open_api_operation(:get_finality_checkpoints), do: ApiSpec.spec().paths["/eth/v1/beacon/states/{state_id}/finality_checkpoints"].get + def open_api_operation(:get_headers_by_block), + do: ApiSpec.spec().paths["/eth/v1/beacon/headers/{block_id}"].get + @spec get_genesis(Plug.Conn.t(), any) :: Plug.Conn.t() def get_genesis(conn, _params) do conn |> json(%{ "data" => %{ - "genesis_time" => StoreDb.fetch_genesis_time!(), + "genesis_time" => StoreDb.fetch_genesis_time!() |> Integer.to_string(), "genesis_validators_root" => ChainSpec.get_genesis_validators_root() |> Utils.hex_encode(), "genesis_fork_version" => ChainSpec.get("GENESIS_FORK_VERSION") |> Utils.hex_encode() @@ -168,18 +171,52 @@ defmodule BeaconApi.V1.BeaconController do finalized: finalized, data: %{ previous_justified: %{ - epoch: previous_justified_checkpoint.epoch, + epoch: previous_justified_checkpoint.epoch |> Integer.to_string(), root: Utils.hex_encode(previous_justified_checkpoint.root) }, current_justified: %{ - epoch: current_justified_checkpoint.epoch, + epoch: current_justified_checkpoint.epoch |> Integer.to_string(), root: Utils.hex_encode(current_justified_checkpoint.root) }, finalized: %{ - epoch: finalized_checkpoint.epoch, + epoch: finalized_checkpoint.epoch |> Integer.to_string(), root: Utils.hex_encode(finalized_checkpoint.root) } } }) end + + @spec get_headers_by_block(Plug.Conn.t(), any) :: Plug.Conn.t() + def get_headers_by_block(conn, %{block_id: "head"}) do + {:ok, store} = StoreDb.fetch_store() + head_root = store.head_root + %{signed_block: %{message: message, signature: signature}} = Blocks.get_block_info(head_root) + + conn + # TODO: This is a placeholder, a minimum implementation to make assertoor run + |> json(%{ + execution_optimistic: false, + + # This is obviously false for the head, but should be derived + finalized: false, + data: %{ + root: head_root |> Utils.hex_encode(), + + # This needs to be derived + canonical: true, + header: %{ + message: %{ + slot: message.slot |> Integer.to_string(), + proposer_index: message.proposer_index |> Integer.to_string(), + parent_root: message.parent_root |> Utils.hex_encode(), + state_root: message.state_root |> Utils.hex_encode(), + body_root: SszEx.hash_tree_root!(message.body) |> Utils.hex_encode() + }, + signature: signature |> Utils.hex_encode() + } + } + }) + end + + def get_headers_by_block(conn, _params), do: conn |> ErrorController.not_found(nil) end diff --git a/lib/beacon_api/controllers/v1/config_controller.ex b/lib/beacon_api/controllers/v1/config_controller.ex new file mode 100644 index 000000000..727442ea0 --- /dev/null +++ b/lib/beacon_api/controllers/v1/config_controller.ex @@ -0,0 +1,61 @@ +defmodule BeaconApi.V1.ConfigController do + use BeaconApi, :controller + require Logger + + alias BeaconApi.ApiSpec + alias BeaconApi.Utils + + plug(OpenApiSpex.Plug.CastAndValidate, json_render_error_v2: true) + + @chain_spec_removed_keys [ + "ATTESTATION_SUBNET_COUNT", + "KZG_COMMITMENT_INCLUSION_PROOF_DEPTH", + "UPDATE_TIMEOUT" + ] + @chain_spec_renamed_keys [ + {"MAXIMUM_GOSSIP_CLOCK_DISPARITY", "MAXIMUM_GOSSIP_CLOCK_DISPARITY_MILLIS"} + ] + @chain_spec_hex_fields [ + "TERMINAL_BLOCK_HASH", + "GENESIS_FORK_VERSION", + "ALTAIR_FORK_VERSION", + "BELLATRIX_FORK_VERSION", + "CAPELLA_FORK_VERSION", + "DENEB_FORK_VERSION", + "ELECTRA_FORK_VERSION", + "DEPOSIT_CONTRACT_ADDRESS", + "MESSAGE_DOMAIN_INVALID_SNAPPY", + "MESSAGE_DOMAIN_VALID_SNAPPY" + ] + + # NOTE: this function is required by OpenApiSpex, and should return the information + # of each specific endpoint. We just return the specific entry from the parsed spec. + def open_api_operation(:get_spec), + do: ApiSpec.spec().paths["/eth/v1/config/spec"].get + + # TODO: This is still an incomplete implementation, it should return some constants + # along with the chain spec. It's enough for assertoor. + @spec get_spec(Plug.Conn.t(), any) :: Plug.Conn.t() + def get_spec(conn, _params), do: json(conn, %{"data" => chain_spec()}) + + defp chain_spec() do + ChainSpec.get_all() + |> Map.drop(@chain_spec_removed_keys) + |> rename_keys(@chain_spec_renamed_keys) + |> Map.new(fn + {k, v} when is_integer(v) -> {k, Integer.to_string(v)} + {k, v} when k in @chain_spec_hex_fields -> {k, Utils.hex_encode(v)} + {k, v} -> {k, v} + end) + end + + defp rename_keys(config, renamed_keys) do + renamed_keys + |> Enum.reduce(config, fn {old_key, new_key}, config -> + case Map.get(config, old_key) do + nil -> config + value -> Map.put_new(config, new_key, value) |> Map.delete(old_key) + end + end) + end +end diff --git a/lib/beacon_api/controllers/v1/node_controller.ex b/lib/beacon_api/controllers/v1/node_controller.ex index 334501d83..5a7ad4768 100644 --- a/lib/beacon_api/controllers/v1/node_controller.ex +++ b/lib/beacon_api/controllers/v1/node_controller.ex @@ -19,12 +19,21 @@ defmodule BeaconApi.V1.NodeController do def open_api_operation(:version), do: ApiSpec.spec().paths["/eth/v1/node/version"].get + def open_api_operation(:syncing), + do: ApiSpec.spec().paths["/eth/v1/node/syncing"].get + + def open_api_operation(:peers), + do: ApiSpec.spec().paths["/eth/v1/node/peers"].get + @spec health(Plug.Conn.t(), any) :: Plug.Conn.t() def health(conn, params) do - # TODO: respond with syncing status if we're still syncing - _syncing_status = Map.get(params, :syncing_status, 206) + %{syncing?: syncing?} = Libp2pPort.sync_status() + + syncing_status = if syncing?, do: Map.get(params, :syncing_status, 206), else: 200 - send_resp(conn, 200, "") + send_resp(conn, syncing_status, "") + rescue + _ -> send_resp(conn, 503, "") end @spec identity(Plug.Conn.t(), any) :: Plug.Conn.t() @@ -62,4 +71,25 @@ defmodule BeaconApi.V1.NodeController do } }) end + + @spec syncing(Plug.Conn.t(), any) :: Plug.Conn.t() + def syncing(conn, _params) do + %{ + syncing?: is_syncing, + optimistic?: is_optimistic, + el_offline?: el_offline, + head_slot: head_slot, + sync_distance: sync_distance + } = Libp2pPort.sync_status() + + json(conn, %{ + "data" => %{ + "is_syncing" => is_syncing, + "is_optimistic" => is_optimistic, + "el_offline" => el_offline, + "head_slot" => head_slot |> Integer.to_string(), + "sync_distance" => sync_distance |> Integer.to_string() + } + }) + end end diff --git a/lib/beacon_api/helpers.ex b/lib/beacon_api/helpers.ex index fc5511bc5..8db8b2822 100644 --- a/lib/beacon_api/helpers.ex +++ b/lib/beacon_api/helpers.ex @@ -154,10 +154,14 @@ defmodule BeaconApi.Helpers do @spec finality_checkpoint_by_id(state_id()) :: {:ok, finality_info()} | {:error, String.t()} | :not_found | :empty_slot | :invalid_id def finality_checkpoint_by_id(id) do + empty_checkpoint = %Types.Checkpoint{epoch: 0, root: <<0::256>>} + with {:ok, {state, optimistic, finalized}} <- state_by_state_id(id) do - {:ok, - {state.previous_justified_checkpoint, state.current_justified_checkpoint, - state.finalized_checkpoint, optimistic, finalized}} + previous_justified_ck = Map.get(state, :previous_justified_checkpoint, empty_checkpoint) + current_justified_ck = Map.get(state, :current_justified_checkpoint, empty_checkpoint) + finalized_ck = Map.get(state, :finalized_checkpoint, empty_checkpoint) + + {:ok, {previous_justified_ck, current_justified_ck, finalized_ck, optimistic, finalized}} end end diff --git a/lib/beacon_api/router.ex b/lib/beacon_api/router.ex index 526c0d7da..05020e6cc 100644 --- a/lib/beacon_api/router.ex +++ b/lib/beacon_api/router.ex @@ -1,9 +1,11 @@ defmodule BeaconApi.Router do use BeaconApi, :router + require Logger pipeline :api do plug(:accepts, ["json", "sse"]) plug(OpenApiSpex.Plug.PutApiSpec, module: BeaconApi.ApiSpec) + plug(:log_requests) end # Ethereum API Version 1 @@ -15,12 +17,19 @@ defmodule BeaconApi.Router do get("/states/:state_id/root", BeaconController, :get_state_root) get("/blocks/:block_id/root", BeaconController, :get_block_root) get("/states/:state_id/finality_checkpoints", BeaconController, :get_finality_checkpoints) + get("/headers/:block_id", BeaconController, :get_headers_by_block) + end + + scope "/config" do + get("/spec", ConfigController, :get_spec) end scope "/node" do get("/health", NodeController, :health) get("/identity", NodeController, :identity) get("/version", NodeController, :version) + get("/syncing", NodeController, :syncing) + get("/peers", NodeController, :peers) end scope "/events" do @@ -44,4 +53,16 @@ defmodule BeaconApi.Router do # Catch-all route outside of any scope match(:*, "/*path", BeaconApi.ErrorController, :not_found) + + defp log_requests(conn, _opts) do + base_message = "[BeaconAPI Router] Processing request: #{conn.method} - #{conn.request_path}" + query = if conn.query_params != %{}, do: "Query: #{inspect(conn.query_params)}", else: "" + body = if conn.body_params != %{}, do: "Body: #{inspect(conn.body_params)}", else: "" + + [base_message, query, body] + |> Enum.join("\n\t") + |> Logger.info() + + conn + end end diff --git a/lib/chain_spec/chain_spec.ex b/lib/chain_spec/chain_spec.ex index 30fc3682e..9bc89b6cd 100644 --- a/lib/chain_spec/chain_spec.ex +++ b/lib/chain_spec/chain_spec.ex @@ -19,6 +19,8 @@ defmodule ChainSpec do # NOTE: this only works correctly for Capella def get(name), do: get_config().get(name) + def get_all(), do: get_config().get_all() + def get_genesis_validators_root() do Application.fetch_env!(:lambda_ethereum_consensus, __MODULE__) |> Keyword.fetch!(:genesis_validators_root) diff --git a/lib/lambda_ethereum_consensus/beacon/sync_blocks.ex b/lib/lambda_ethereum_consensus/beacon/sync_blocks.ex index 6050bbb73..cc009cf17 100644 --- a/lib/lambda_ethereum_consensus/beacon/sync_blocks.ex +++ b/lib/lambda_ethereum_consensus/beacon/sync_blocks.ex @@ -20,11 +20,10 @@ defmodule LambdaEthereumConsensus.Beacon.SyncBlocks do finish, each block of those responses will be sent to libp2p port module individually using Libp2pPort.add_block/1. """ - @spec run() :: non_neg_integer() - def run() do - %{head_slot: head_slot} = ForkChoice.get_current_status_message() + @spec run(Types.Store.t()) :: non_neg_integer() + def run(%{head_slot: head_slot} = store) do initial_slot = head_slot + 1 - last_slot = ForkChoice.get_current_chain_slot() + last_slot = ForkChoice.get_current_slot(store) # If we're around genesis, we consider ourselves synced if last_slot <= 0 do diff --git a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex index d473b37b6..0fc522586 100644 --- a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex +++ b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex @@ -125,8 +125,7 @@ defmodule LambdaEthereumConsensus.ForkChoice do @doc """ Get the current chain slot based on the system time. - There are just 2 uses of this function outside this module: - - At the begining of SyncBlocks.run/1 function, to get the head slot + There is just 1 use of this function outside this module: - In the Helpers.block_root_by_block_id/1 function """ @spec get_current_chain_slot() :: Types.slot() diff --git a/lib/libp2p_port.ex b/lib/libp2p_port.ex index ef30d1f68..3a1e9c2e0 100644 --- a/lib/libp2p_port.ex +++ b/lib/libp2p_port.ex @@ -384,6 +384,21 @@ defmodule LambdaEthereumConsensus.Libp2pPort do GenServer.cast(pid, {:error_downloading_chunk, range, reason}) end + @doc """ + Returns the current sync status. + """ + @spec sync_status(pid | atom()) :: %{ + syncing?: boolean(), + optimistic?: boolean(), + el_offline?: boolean(), + head_slot: Types.slot(), + sync_distance: non_neg_integer(), + blocks_remaining: non_neg_integer() + } + def sync_status(pid \\ __MODULE__) do + GenServer.call(pid, :sync_status) + end + ######################## ### GenServer Callbacks ######################## @@ -513,8 +528,8 @@ defmodule LambdaEthereumConsensus.Libp2pPort do end @impl GenServer - def handle_info(:sync_blocks, state) do - blocks_to_download = SyncBlocks.run() + def handle_info(:sync_blocks, %{store: store} = state) do + blocks_to_download = SyncBlocks.run(store) new_state = state |> Map.put(:blocks_remaining, blocks_to_download) |> subscribe_if_no_blocks() @@ -575,6 +590,30 @@ defmodule LambdaEthereumConsensus.Libp2pPort do {:reply, :ok, %{state | validator_set: validator_set}} end + @impl GenServer + def handle_call( + :sync_status, + _from, + %{syncing: syncing?, store: %Types.Store{} = store} = state + ) do + # TODO: (#1325) This is not the final implementation, we are lacking the el check, + # this is just in place for start using assertoor. + head_slot = store.head_slot + current_slot = ForkChoice.get_current_slot(store) + distance = current_slot - head_slot + + result = %{ + syncing?: syncing?, + optimistic?: syncing?, + el_offline?: false, + head_slot: store.head_slot, + sync_distance: distance, + blocks_remaining: Map.get(state, :blocks_remaining) + } + + {:reply, result, state} + end + ###################### ### PRIVATE FUNCTIONS ###################### diff --git a/network_params.yaml b/network_params.yaml index 13618b8b9..06e6f07c0 100644 --- a/network_params.yaml +++ b/network_params.yaml @@ -14,5 +14,23 @@ participants: validator_count: 32 cl_max_mem: 4096 keymanager_enabled: true -network_params: - preset: minimal +# Uncomment the following lines to run the the network with the minimal preset (which is various times faster) +# network_params: +# preset: minimal +additional_services: + - tx_spammer + - blob_spammer + - dora + - beacon_metrics_gazer + - prometheus_grafana +# Uncomment the following lines to run the the network with the assertoor tests +# - assertoor + +# assertoor_params: +# run_stability_check: false +# run_block_proposal_check: false +# tests: +# - https://raw.githubusercontent.com/lambdaclass/lambda_ethereum_consensus/refs/heads/main/.github/config/assertoor/cl-stability-check.yml + +# tx_spammer_params: +# tx_spammer_extra_args: ["--txcount=3", "--accounts=80"] diff --git a/test/unit/beacon_api/beacon_api_v1_test.exs b/test/unit/beacon_api/beacon_api_v1_test.exs index 75e4323e5..2f4740647 100644 --- a/test/unit/beacon_api/beacon_api_v1_test.exs +++ b/test/unit/beacon_api/beacon_api_v1_test.exs @@ -6,6 +6,8 @@ defmodule Unit.BeaconApiTest.V1 do alias BeaconApi.Router alias BeaconApi.Utils alias LambdaEthereumConsensus.ForkChoice + alias LambdaEthereumConsensus.Libp2pPort + alias LambdaEthereumConsensus.P2P.Metadata alias LambdaEthereumConsensus.Store.BlockDb alias LambdaEthereumConsensus.Store.Db alias LambdaEthereumConsensus.Store.StoreDb @@ -107,15 +109,15 @@ defmodule Unit.BeaconApiTest.V1 do "execution_optimistic" => true, "data" => %{ "previous_justified" => %{ - "epoch" => beacon_state.previous_justified_checkpoint.epoch, + "epoch" => Integer.to_string(beacon_state.previous_justified_checkpoint.epoch), "root" => Utils.hex_encode(beacon_state.previous_justified_checkpoint.root) }, "current_justified" => %{ - "epoch" => beacon_state.current_justified_checkpoint.epoch, + "epoch" => Integer.to_string(beacon_state.current_justified_checkpoint.epoch), "root" => Utils.hex_encode(beacon_state.current_justified_checkpoint.root) }, "finalized" => %{ - "epoch" => beacon_state.finalized_checkpoint.epoch, + "epoch" => Integer.to_string(beacon_state.finalized_checkpoint.epoch), "root" => Utils.hex_encode(beacon_state.finalized_checkpoint.root) } } @@ -134,7 +136,7 @@ defmodule Unit.BeaconApiTest.V1 do test "get genesis data" do expected_response = %{ "data" => %{ - "genesis_time" => StoreDb.fetch_genesis_time!(), + "genesis_time" => StoreDb.fetch_genesis_time!() |> Integer.to_string(), "genesis_validators_root" => ChainSpec.get_genesis_validators_root() |> Utils.hex_encode(), "genesis_fork_version" => ChainSpec.get("GENESIS_FORK_VERSION") |> Utils.hex_encode() @@ -149,16 +151,46 @@ defmodule Unit.BeaconApiTest.V1 do end test "node health" do + now = :os.system_time(:second) + patch(Libp2pPort, :on_tick, fn _time, state -> state end) + + start_link_supervised!( + {Libp2pPort, + genesis_time: now - 24, store: %Store{genesis_time: now - 24, time: now, head_slot: 1}} + ) + conn = conn(:get, "/eth/v1/node/health", nil) |> Router.call(@opts) assert conn.state == :sent - assert conn.status == 200 + assert conn.status == 206 assert conn.resp_body == "" end - test "node identity" do - alias LambdaEthereumConsensus.Libp2pPort - alias LambdaEthereumConsensus.P2P.Metadata + test "node syncing" do + now = :os.system_time(:second) + patch(Libp2pPort, :on_tick, fn _time, state -> state end) + + start_link_supervised!( + {Libp2pPort, + genesis_time: now - 24, store: %Store{genesis_time: now - 24, time: now, head_slot: 1}} + ) + + expected_response = %{ + "data" => %{ + "is_syncing" => true, + "is_optimistic" => true, + "el_offline" => false, + "head_slot" => "1", + "sync_distance" => "1" + } + } + conn = conn(:get, "/eth/v1/node/syncing", nil) |> Router.call(@opts) + assert conn.state == :sent + assert conn.status == 200 + assert Jason.decode!(conn.resp_body) == expected_response + end + + test "node identity" do patch(BeaconApi.EventPubSub, :publish, fn _, _ -> :ok end) patch(ForkChoice, :get_fork_version, fn -> ChainSpec.get("DENEB_FORK_VERSION") end)