fix(explorer): fetch_batch_data_pointer in explorer reads the entire response body without any limitation (#1198)

Co-authored-by: Urix <[email protected]>

Author: avilagaston9

explorer/.env.dev

@@ -20,3 +20,5 @@ DEBUG_ERRORS=true
 
 # Operator version tracker API
 TRACKER_API_URL=http://localhost:3030
+
+MAX_BATCH_SIZE=268435456 # 256 MiB

explorer/lib/explorer/contract_managers/aligned_layer_service_manager.ex

@@ -136,7 +136,8 @@ defmodule AlignedLayerServiceManager do
       proof_hashes: nil,
       fee_per_proof: BatcherPaymentServiceManager.get_fee_per_proof(%{merkle_root: created_batch.batchMerkleRoot}),
       sender_address: Utils.string_to_bytes32(created_batch.senderAddress),
-      max_aggregator_fee: created_batch.maxAggregatorFee
+      max_aggregator_fee: created_batch.maxAggregatorFee,
+      is_valid: true # set to false later if a process determines it is invalid
     }
   end
 
@@ -166,7 +167,8 @@ defmodule AlignedLayerServiceManager do
           fee_per_proof: unverified_batch.fee_per_proof,
           proof_hashes: nil,
           sender_address: unverified_batch.sender_address,
-          max_aggregator_fee: unverified_batch.max_aggregator_fee
+          max_aggregator_fee: unverified_batch.max_aggregator_fee,
+          is_valid: true # set to false later if a process determines it is invalid
         }
     end
   end

explorer/lib/explorer/models/batch_structs.ex

@@ -32,7 +32,8 @@ defmodule BatchDB do
     :proof_hashes,
     :fee_per_proof,
     :sender_address,
-    :max_aggregator_fee
+    :max_aggregator_fee,
+    :is_valid
   ]
   defstruct [
     :merkle_root,
@@ -48,6 +49,7 @@ defmodule BatchDB do
     :proof_hashes,
     :fee_per_proof,
     :sender_address,
-    :max_aggregator_fee
+    :max_aggregator_fee,
+    :is_valid
   ]
 end

explorer/lib/explorer/models/batches.ex

@@ -18,15 +18,16 @@ defmodule Batches do
     field :fee_per_proof, :integer
     field :sender_address, :binary
     field :max_aggregator_fee, :decimal
+    field :is_valid, :boolean, default: true
 
     timestamps()
   end
 
   @doc false
   def changeset(new_batch, updates) do
     new_batch
-    |> cast(updates, [:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :submission_timestamp, :response_block_number, :response_transaction_hash, :response_timestamp, :data_pointer, :fee_per_proof, :sender_address, :max_aggregator_fee])
-    |> validate_required([:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :fee_per_proof, :sender_address])
+    |> cast(updates, [:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :submission_timestamp, :response_block_number, :response_transaction_hash, :response_timestamp, :data_pointer, :fee_per_proof, :sender_address, :max_aggregator_fee, :is_valid])
+    |> validate_required([:merkle_root, :amount_of_proofs, :is_verified, :submission_block_number, :submission_transaction_hash, :fee_per_proof, :sender_address, :is_valid])
     |> validate_format(:merkle_root, ~r/0x[a-fA-F0-9]{64}/)
     |> unique_constraint(:merkle_root)
     |> validate_number(:amount_of_proofs, greater_than: 0)
@@ -37,6 +38,7 @@ defmodule Batches do
     |> validate_format(:response_transaction_hash, ~r/0x[a-fA-F0-9]{64}/)
     |> validate_number(:max_aggregator_fee, greater_than: 0)
     |> validate_number(:fee_per_proof, greater_than_or_equal_to: 0)
+    |> validate_inclusion(:is_valid, [true, false])
   end
 
   def cast_to_batches(%BatchDB{} = batch_db) do
@@ -53,7 +55,8 @@ defmodule Batches do
       data_pointer: batch_db.data_pointer,
       fee_per_proof: batch_db.fee_per_proof,
       sender_address: batch_db.sender_address,
-      max_aggregator_fee: batch_db.max_aggregator_fee
+      max_aggregator_fee: batch_db.max_aggregator_fee,
+      is_valid: batch_db.is_valid
     }
   end
 
@@ -113,7 +116,7 @@ defmodule Batches do
     threshold_datetime = DateTime.utc_now() |> DateTime.add(-43200, :second) # 12 hours ago
 
     query = from(b in Batches,
-    where: b.is_verified == false and b.submission_timestamp > ^threshold_datetime,
+    where: b.is_valid == true and b.is_verified == false and b.submission_timestamp > ^threshold_datetime,
     select: b)
 
     Explorer.Repo.all(query)
@@ -193,5 +196,4 @@ defmodule Batches do
         end
     end
   end
-
 end

explorer/lib/explorer/periodically.ex

@@ -45,6 +45,7 @@ defmodule Explorer.Periodically do
 
     run_every_n_iterations = 8
     new_count = rem(count + 1, run_every_n_iterations)
+
     if new_count == 0 do
       Task.start(&process_unverified_batches/0)
     end
@@ -79,25 +80,19 @@ defmodule Explorer.Periodically do
       {:ok, lock} ->
         "Processing batch: #{batch.merkle_root}" |> Logger.debug()
 
-        {batch_changeset, proofs} =
-          batch
-          |> Utils.extract_info_from_data_pointer()
-          |> Batches.generate_changesets()
-
-        Batches.insert_or_update(batch_changeset, proofs)
-        |> case do
-          {:ok, _} ->
-            PubSub.broadcast(Explorer.PubSub, "update_views", %{
-              eth_usd:
-                case EthConverter.get_eth_price_usd() do
-                  {:ok, eth_usd_price} -> eth_usd_price
-                  {:error, _error} -> :empty
-                end
-            })
-
-          {:error, error} ->
-            Logger.error("Some error in DB operation, not broadcasting update_views: #{inspect(error)}")
-
+        with {:ok, updated_batch} <- Utils.process_batch(batch),
+             {batch_changeset, proofs} <- Batches.generate_changesets(updated_batch),
+             {:ok, _} <- Batches.insert_or_update(batch_changeset, proofs) do
+          PubSub.broadcast(Explorer.PubSub, "update_views", %{
+            eth_usd:
+              case EthConverter.get_eth_price_usd() do
+                {:ok, eth_usd_price} -> eth_usd_price
+                {:error, _error} -> :empty
+              end
+          })
+        else
+          {:error, reason} ->
+            Logger.error("Error processing batch #{batch.merkle_root}. Error: #{inspect(reason)}")
           # no changes in DB
           nil ->
             nil

explorer/lib/explorer_web/components/core_components.ex

@@ -417,15 +417,15 @@ defmodule ExplorerWeb.CoreComponents do
   end
 
   @doc """
-    Renders a dynamic badge compoent.
+    Renders a dynamic badge component.
   """
   attr :class, :string, default: nil
   attr :status, :boolean, default: true
-  attr :falsy_text, :string, default: "Pending"
-  attr :truthy_text, :string, default: "Verified"
+  attr :falsy_text, :string
+  attr :truthy_text, :string
   slot :inner_block, default: nil
 
-  def dynamic_badge(assigns) do
+  def dynamic_badge_boolean(assigns) do
     ~H"""
     <.badge
       variant={
@@ -449,6 +449,39 @@ defmodule ExplorerWeb.CoreComponents do
     """
   end
 
+  @doc """
+    Renders a dynamic badge component for the batcher.
+  """
+  attr :class, :string, default: nil
+  attr :status, :atom
+  slot :inner_block, default: nil
+
+  def dynamic_badge_for_batcher(assigns) do
+    ~H"""
+    <.badge
+      variant={
+        case @status do
+          :invalid -> "destructive"
+          :verified -> "accent"
+          :pending -> "foreground"
+        end
+      }
+      class={
+        classes([
+          @class
+        ])
+      }
+    >
+      <%= case @status do
+        :invalid -> "Invalid"
+        :verified -> "Verified"
+        :pending -> "Pending"
+      end %>
+      <%= render_slot(@inner_block) %>
+    </.badge>
+    """
+  end
+
   @doc """
   Renders an input with label and error messages.
 

explorer/lib/explorer_web/live/pages/batch/index.html.heex

@@ -25,7 +25,7 @@
         <h3>
           Status:
         </h3>
-        <.dynamic_badge class="w-fit" status={@current_batch.is_verified} />
+        <.dynamic_badge_for_batcher class="w-fit" status={Helpers.get_batch_status(@current_batch)} />
       </div>
       <div>
         <h3>

explorer/lib/explorer_web/live/pages/batches/index.html.heex

@@ -14,7 +14,7 @@
         </.link>
       </:col>
       <:col :let={batch} label="Status">
-        <.dynamic_badge status={batch.is_verified} />
+        <.dynamic_badge_for_batcher status={Helpers.get_batch_status(batch)} />
       </:col>
       <:col :let={batch} label="Age">
         <span class="md:px-0" title={batch.submission_timestamp}>

explorer/lib/explorer_web/live/pages/operators/index.ex

@@ -83,7 +83,7 @@ defmodule ExplorerWeb.Operators.Index do
             <%= operator.total_stake |> EthConverter.wei_to_eth(2) |> Helpers.format_number() %> ETH
           </:col>
           <:col :let={operator} label="Status">
-            <.dynamic_badge status={operator.is_active} truthy_text="Active" falsy_text="Inactive" />
+            <.dynamic_badge_boolean status={operator.is_active} truthy_text="Active" falsy_text="Inactive" />
           </:col>
         </.table>
       <% else %>