feat(l1): run daily snapsync checks with trie validations (debug-assertions) (#5768)

**Motivation**

We want to make sure that the snapsyncs running in loop, not just finish
but their state is validated

**Description**

This PR acomplish 2 things:
- It makes `debug-assertions` easily available for docker
- It change how the daily snapsyncs work, building locally instead of
using the main image and running with debug-assertions
- timeouts are now longer

---------

Co-authored-by: Edgar <[email protected]>

Author: rodrigo-o

.github/actions/snapsync-run/action.yml

@@ -15,6 +15,14 @@ inputs:
     description: Ethrex Docker image tag.
     required: false
     default: main
+  build_local:
+    description: Build ethrex image locally instead of using a pre-built image.
+    required: false
+    default: "false"
+  build_profile:
+    description: Cargo profile to use when building locally (e.g., release, release-with-debug-assertions).
+    required: false
+    default: release
   cl_type:
     description: Consensus layer type (lighthouse, prysm, etc).
     required: false
@@ -34,12 +42,25 @@ runs:
       shell: bash
       run: docker image rm -f ${{ inputs.ethrex_image }}:${{ inputs.ethrex_tag }} || true
 
+    - name: Build local ethrex image
+      if: inputs.build_local == 'true'
+      shell: bash
+      env:
+        BUILD_PROFILE: ${{ inputs.build_profile }}
+        IMAGE_TAG: ${{ inputs.ethrex_tag }}
+      run: |
+        echo "Building ethrex with profile: ${BUILD_PROFILE}"
+        docker build \
+          --build-arg PROFILE="${BUILD_PROFILE}" \
+          -t ethrex-local:${IMAGE_TAG} \
+          -f Dockerfile .
+
     - name: Generate Kurtosis args
       shell: bash
       env:
         NETWORK_NAME: ${{ inputs.network }}
         TIMEOUT: ${{ inputs.timeout }}
-        ETHREX_IMAGE: ${{ inputs.ethrex_image }}
+        ETHREX_IMAGE: ${{ inputs.build_local == 'true' && 'ethrex-local' || inputs.ethrex_image }}
         ETHREX_TAG: ${{ inputs.ethrex_tag }}
         CL_TYPE: ${{ inputs.cl_type }}
         CL_IMAGE: ${{ inputs.cl_image }}
@@ -91,11 +112,23 @@ runs:
     - name: Summarize ethrex version
       if: ${{ always() }}
       shell: bash
+      env:
+        BUILD_LOCAL: ${{ inputs.build_local }}
+        BUILD_PROFILE: ${{ inputs.build_profile }}
+        ETHREX_IMAGE: ${{ inputs.ethrex_image }}
+        ETHREX_TAG: ${{ inputs.ethrex_tag }}
       run: |
-        image="${{ inputs.ethrex_image }}:${{ inputs.ethrex_tag }}"
+        if [ "$BUILD_LOCAL" = "true" ]; then
+          image="ethrex-local:${ETHREX_TAG}"
+        else
+          image="${ETHREX_IMAGE}:${ETHREX_TAG}"
+        fi
         version="$(docker run --rm "$image" --version 2>/dev/null || true)"
         {
           echo "### Ethrex Version"
           echo ""
           echo "- ${version:-unavailable}"
+          if [ "$BUILD_LOCAL" = "true" ]; then
+            echo "- Build Profile: ${BUILD_PROFILE}"
+          fi
         } >> "${GITHUB_STEP_SUMMARY}"

.github/workflows/daily_snapsync.yaml

@@ -14,6 +14,10 @@ on:
         description: "Network name (hoodi or sepolia)"
         required: false
         default: "hoodi"
+      build_profile:
+        description: "Cargo build profile (release or release-with-debug-assertions)"
+        required: false
+        default: "release-with-debug-assertions"
 
 permissions:
   contents: read
@@ -27,23 +31,24 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.targets.outputs.matrix }}
+      build_profile: ${{ steps.build_opts.outputs.build_profile }}
     steps:
       - id: targets
         run: |
           event_name="${GITHUB_EVENT_NAME}"
           if [[ "$event_name" == "schedule" ]]; then
-            json='[{"network":"hoodi","timeout":"40m"},{"network":"sepolia","timeout":"2h30m"}]'
+            json='[{"network":"hoodi","timeout":"1h"},{"network":"sepolia","timeout":"3h30m"}]'
           elif [[ "$event_name" == "pull_request" ]]; then
-            json='[{"network":"hoodi","timeout":"40m"}]'
+            json='[{"network":"hoodi","timeout":"1h"}]'
           else
             network=$(jq -r '.inputs.network // empty' "$GITHUB_EVENT_PATH")
 
             case "$network" in
               hoodi)
-                json='[{"network":"hoodi","timeout":"40m"}]'
+                json='[{"network":"hoodi","timeout":"1h"}]'
                 ;;
               sepolia)
-                json='[{"network":"sepolia","timeout":"2h30m"}]'
+                json='[{"network":"sepolia","timeout":"3h30m"}]'
                 ;;
               *)
                 echo "::error::Unsupported network value '$network'. Allowed values: hoodi, sepolia."
@@ -52,6 +57,17 @@ jobs:
             esac
           fi
           echo "matrix=$json" >> "$GITHUB_OUTPUT"
+      - id: build_opts
+        run: |
+          event_name="${GITHUB_EVENT_NAME}"
+          # Default to release-with-debug-assertions for scheduled runs and PRs
+          # Manual runs can specify a different profile
+          if [[ "$event_name" == "workflow_dispatch" ]]; then
+            profile=$(jq -r '.inputs.build_profile // "release-with-debug-assertions"' "$GITHUB_EVENT_PATH")
+          else
+            profile="release-with-debug-assertions"
+          fi
+          echo "build_profile=$profile" >> "$GITHUB_OUTPUT"
 
   sync-lighthouse:
     needs: prepare
@@ -78,6 +94,8 @@ jobs:
           timeout: ${{ matrix.timeout }}
           cl_type: lighthouse
           cl_image: "sigp/lighthouse:v8.0.1"
+          build_local: "true"
+          build_profile: ${{ needs.prepare.outputs.build_profile }}
 
       - name: Post run on slack
         if: always()
@@ -117,6 +135,8 @@ jobs:
           timeout: ${{ matrix.timeout }}
           cl_type: prysm
           cl_image: "gcr.io/offchainlabs/prysm/beacon-chain:v7.1.0"
+          build_local: "true"
+          build_profile: ${{ needs.prepare.outputs.build_profile }}
 
       - name: Post run on slack
         if: always()

Cargo.toml

@@ -48,6 +48,10 @@ codegen-units = 1
 inherits = "release"
 debug = 2
 
+[profile.release-with-debug-assertions]
+inherits = "release"
+debug-assertions = true
+
 [workspace.dependencies]
 ethrex-blockchain = { path = "./crates/blockchain", default-features = false }
 ethrex-common = { path = "./crates/common", default-features = false }

Dockerfile

@@ -33,7 +33,10 @@ RUN cargo chef prepare --recipe-path recipe.json
 # previous stage has changed, which only happens when dependencies change.
 FROM chef AS builder
 
-# Optional build flags
+# Build configuration
+# PROFILE: Cargo profile to use (release, release-with-debug-assertions, etc.)
+# BUILD_FLAGS: Additional cargo flags (features, etc.)
+ARG PROFILE="release"
 ARG BUILD_FLAGS=""
 
 COPY --from=planner /ethrex/recipe.json recipe.json
@@ -60,7 +63,11 @@ COPY fixtures ./fixtures
 COPY .cargo/ ./.cargo
 
 ENV COMPILE_CONTRACTS=true
-RUN cargo build --release $BUILD_FLAGS
+
+RUN cargo build --profile $PROFILE $BUILD_FLAGS
+
+RUN mkdir -p /ethrex/bin && \
+    cp /ethrex/target/${PROFILE}/ethrex /ethrex/bin/ethrex
 
 # --- Final Image ---
 # Copy the ethrex binary into a minimalist image to reduce bloat size.
@@ -71,7 +78,7 @@ WORKDIR /usr/local/bin
 RUN apt-get update && apt-get install -y --no-install-recommends libssl3
 
 COPY cmd/ethrex/networks ./cmd/ethrex/networks
-COPY --from=builder /ethrex/target/release/ethrex .
+COPY --from=builder /ethrex/bin/ethrex .
 
 # Common ports:
 # -  8545: RPC

docs/developers/ci/daily_snapsync.md

@@ -1,10 +1,27 @@
 # Daily Snapsync Check
 
-Inside [GitHub actions tab](https://github.com/lambdaclass/ethrex/actions) there is a pinned workflow called [Daily Snapsync Check](https://github.com/lambdaclass/ethrex/actions/workflows/daily_snapsync.yaml). As the name suggests, it runs an Ethrex node and attemps to complete a snap sync and catch the head of the chain. If it is not able to finish snapsync before the timeout, the job fails and an [alert](https://github.com/lambdaclass/ethrex/blob/9feefd2e3fd2e8bb2097e5e39e0d20f7315c5880/.github/workflows/common_failure_alerts.yaml#L8) is sent to Slack. The way it works is through an [Assertoor playbook](https://github.com/lambdaclass/ethrex/blob/9feefd2e3fd2e8bb2097e5e39e0d20f7315c5880/.github/config/assertoor/syncing-check.yaml#L1-L17) that checks that the node `eth_syncing` returns `false`.
+Inside [GitHub actions tab](https://github.com/lambdaclass/ethrex/actions) there is a pinned workflow called [Daily Snapsync Check](https://github.com/lambdaclass/ethrex/actions/workflows/daily_snapsync.yaml). As the name suggests, it runs an Ethrex node and attempts to complete a snap sync and catch the head of the chain. If it is not able to finish snapsync before the timeout, the job fails and an [alert](https://github.com/lambdaclass/ethrex/blob/9feefd2e3fd2e8bb2097e5e39e0d20f7315c5880/.github/workflows/common_failure_alerts.yaml#L8) is sent to Slack. The way it works is through an [Assertoor playbook](https://github.com/lambdaclass/ethrex/blob/9feefd2e3fd2e8bb2097e5e39e0d20f7315c5880/.github/config/assertoor/syncing-check.yaml#L1-L17) that checks that the node `eth_syncing` returns `false`.
 
-Currently it runs this check on Sepolia and Hoodi.
+Currently it runs this check on Sepolia and Hoodi with both Lighthouse and Prysm consensus clients.
 
-Apart from being a useful job to catch regressions, it is a good log to see if there were any speedups of slowdowns in terms of time to complete a snap sync.
+## Debug Assertions
 
-Nice to haves:
-- Currently the job runs on the `main` docker image of ethrex. It would be nice to be able to trigger this from a branch so that the workflow is executed by trying to sync the branch ethrex code.
+By default, the workflow builds ethrex with debug assertions enabled (`release-with-debug-assertions` profile). This helps catch potential bugs and invariant violations during the sync process.
+
+The `release-with-debug-assertions` profile:
+- Inherits all release optimizations
+- Enables `debug_assert!()` macros to run
+
+## Triggering the Workflow
+
+The workflow can be triggered in three ways:
+
+1. **Scheduled**: Runs every 6 hours automatically on both Hoodi and Sepolia networks
+2. **Pull Request**: Runs on PRs that modify the workflow file (Hoodi only)
+3. **Manual Dispatch**: Can be triggered manually via GitHub Actions UI with options:
+   - `network`: Choose between `hoodi` or `sepolia`
+   - `build_profile`: Choose between `release` or `release-with-debug-assertions` (default: `release-with-debug-assertions`)
+
+## Use Cases
+
+Apart from being a useful job to catch regressions, it is a good log to see if there were any speedups or slowdowns in terms of time to complete a snap sync. With debug assertions enabled, it also serves as an additional validation layer to catch bugs that might not manifest as crashes in release builds.

docs/developers/installing.md

@@ -50,8 +50,17 @@ The Dockerfile is located at the root of the repository and can be built by runn
 docker build -t ethrex .
 ```
 
-The `BUILD_FLAGS` argument can be used to pass flags to cargo, for example
+Build arguments:
+- `PROFILE`: Cargo profile to use (default: `release`). Example: `release-with-debug-assertions`
+- `BUILD_FLAGS`: Additional cargo flags (features, etc.)
 
 ```
-docker build -t ethrex --build-arg BUILD_FLAGS="--features <features>" .
+# Custom profile
+docker build -t ethrex --build-arg PROFILE="release-with-debug-assertions" .
+
+# With features
+docker build -t ethrex --build-arg BUILD_FLAGS="--features l2" .
+
+# Both
+docker build -t ethrex --build-arg PROFILE="release-with-debug-assertions" --build-arg BUILD_FLAGS="--features l2" .
 ```