Merge pull request #7788 from dknopik/into-anchor

Anchor Pre-PR: `unstable`, #7783, #7761, #7016

Author: jimmygchen

.github/CODEOWNERS

@@ -1,2 +1,3 @@
 /beacon_node/network/                @jxs
 /beacon_node/lighthouse_network/     @jxs
+/beacon_node/store/                  @michaelsproul

.github/mergify.yml

@@ -1,8 +1,10 @@
 pull_request_rules:
   - name: Ask to resolve conflict
     conditions:
+      - -closed
       - conflict
       - -author=dependabot[bot]
+      - label=ready-for-review
       - or:
           - -draft # Don't report conflicts on regular draft.
           - and: # Do report conflicts on draft that are scheduled for the next major release.
@@ -12,6 +14,64 @@ pull_request_rules:
       comment:
         message: This pull request has merge conflicts. Could you please resolve them
           @{{author}}? 🙏
+      label:
+        add:
+          - waiting-on-author
+        remove:
+          - ready-for-review
+
+  - name: Ask to resolve CI failures
+    conditions:
+      - -closed
+      - label=ready-for-review
+      - or:
+          - check-skipped=test-suite-success
+          - check-skipped=local-testnet-success
+          - check-failure=test-suite-success
+          - check-failure=local-testnet-success
+    actions:
+      comment:
+        message: Some required checks have failed. Could you please take a look @{{author}}? 🙏
+      label:
+        add:
+          - waiting-on-author
+        remove:
+          - ready-for-review
+
+  - name: Update labels when PR is unblocked
+    conditions:
+      - -closed
+      - -draft
+      - label=waiting-on-author
+      - -conflict
+      # Unfortunately, it doesn't look like there's an easy way to check for PRs pending
+      # CI workflows approvals.
+      - check-success=test-suite-success
+      - check-success=local-testnet-success
+      # Update the label only if there are no more change requests from any reviewers and no unresolved threads.
+      # This rule ensures that a PR with passing CI can be marked as `waiting-on-author`.
+      - "#changes-requested-reviews-by = 0"
+      - "#review-threads-unresolved = 0"
+    actions:
+      label:
+        remove:
+          - waiting-on-author
+        add:
+          - ready-for-review
+
+  - name: Close stale pull request after 30 days of inactivity
+    conditions:
+      - -closed
+      - label=waiting-on-author
+      - updated-at<=30 days ago
+    actions:
+      close:
+        message: >
+          Hi @{{author}}, this pull request has been closed automatically due to 30 days of inactivity.
+          If you’d like to continue working on it, feel free to reopen at any time.
+      label:
+        add:
+          - stale
 
   - name: Approve trivial maintainer PRs
     conditions:

.github/workflows/local-testnet.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Build Docker image
         run: |
-          docker build --build-arg FEATURES=portable -t lighthouse:local .
+          docker build --build-arg FEATURES=portable,spec-minimal -t lighthouse:local .
           docker save lighthouse:local -o lighthouse-docker.tar
 
       - name: Upload Docker image artifact
@@ -52,30 +52,52 @@ jobs:
       - name: Load Docker image
         run: docker load -i lighthouse-docker.tar
 
-      - name: Start local testnet
-        run: ./start_local_testnet.sh -e local -c -b false && sleep 60
+      - name: Start local testnet with Assertoor
+        run: ./start_local_testnet.sh -e local-assertoor -c -a -b false && sleep 60
         working-directory: scripts/local_testnet
 
-      - name: Stop local testnet and dump logs
-        run: ./stop_local_testnet.sh local
-        working-directory: scripts/local_testnet
-
-      - name: Start local testnet with blinded block production
-        run: ./start_local_testnet.sh -e local-blinded -c -p -b false && sleep 60
-        working-directory: scripts/local_testnet
+      - name: Await Assertoor test result
+        id: assertoor_test_result
+        uses: ethpandaops/assertoor-github-action@v1
+        with:
+          kurtosis_enclave_name: local-assertoor
 
       - name: Stop local testnet and dump logs
-        run: ./stop_local_testnet.sh local-blinded
+        run: ./stop_local_testnet.sh local-assertoor
         working-directory: scripts/local_testnet
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-local-testnet
           path: |
             scripts/local_testnet/logs
           retention-days: 3
 
+      - name: Return Assertoor test result
+        shell: bash
+        run: |
+          test_result="${{ steps.assertoor_test_result.outputs.result }}"
+          test_status=$(
+          cat <<"EOF"
+          ${{ steps.assertoor_test_result.outputs.test_overview }}
+          EOF
+          )
+          failed_test_status=$(
+          cat <<"EOF"
+          ${{ steps.assertoor_test_result.outputs.failed_test_details }}
+          EOF
+          )
+          
+          echo "Test Result:  $test_result"
+          echo "$test_status"
+          if ! [ "$test_result" == "success" ]; then
+            echo "Failed Test Task Status:"
+            echo "$failed_test_status"
+            exit 1
+          fi
+
   doppelganger-protection-success-test:
     needs: dockerfile-ubuntu
     runs-on: ubuntu-22.04
@@ -104,6 +126,7 @@ jobs:
         working-directory: scripts/tests
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-doppelganger-protection-success
@@ -139,13 +162,99 @@ jobs:
         working-directory: scripts/tests
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-doppelganger-protection-failure
           path: |
             scripts/local_testnet/logs
           retention-days: 3
 
+  # Tests checkpoint syncing to a live network (current fork) and a running devnet (usually next scheduled fork)
+  checkpoint-sync-test:
+    name: checkpoint-sync-test-${{ matrix.network }}
+    runs-on: ubuntu-latest
+    needs: dockerfile-ubuntu
+    if: contains(github.event.pull_request.labels.*.name, 'syncing')
+    continue-on-error: true
+    strategy:
+      matrix:
+        network: [sepolia, devnet]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli
+          kurtosis analytics disable
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lighthouse-docker
+          path: .
+
+      - name: Load Docker image
+        run: docker load -i lighthouse-docker.tar
+
+      - name: Run the checkpoint sync test script
+        run: |
+          ./checkpoint-sync.sh "sync-${{ matrix.network }}" "checkpoint-sync-config-${{ matrix.network }}.yaml"
+        working-directory: scripts/tests
+
+      - name: Upload logs artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-checkpoint-sync-${{ matrix.network }}
+          path: |
+            scripts/local_testnet/logs
+          retention-days: 3
+
+  # Test syncing from genesis on a local testnet. Aims to cover forward syncing both short and long distances.
+  genesis-sync-test:
+    name: genesis-sync-test-${{ matrix.fork }}-${{ matrix.offline_secs }}s
+    runs-on: ubuntu-latest
+    needs: dockerfile-ubuntu
+    if: contains(github.event.pull_request.labels.*.name, 'syncing')
+    strategy:
+      matrix:
+        fork: [electra, fulu]
+        offline_secs: [120, 300]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli
+          kurtosis analytics disable
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lighthouse-docker
+          path: .
+
+      - name: Load Docker image
+        run: docker load -i lighthouse-docker.tar
+
+      - name: Run the genesis sync test script
+        run: |
+          ./genesis-sync.sh "sync-${{ matrix.fork }}-${{ matrix.offline_secs }}s" "genesis-sync-config-${{ matrix.fork }}.yaml" "${{ matrix.fork }}" "${{ matrix.offline_secs }}"
+        working-directory: scripts/tests
+
+      - name: Upload logs artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-genesis-sync-${{ matrix.fork }}-${{ matrix.offline_secs }}s
+          path: |
+            scripts/local_testnet/logs
+          retention-days: 3
 
   # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether
   # a PR is safe to merge. New jobs should be added here.
@@ -161,4 +270,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check that success job is dependent on all others
-        run: ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success
+        run: |
+          exclude_jobs='checkpoint-sync-test|genesis-sync-test'
+          ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success "$exclude_jobs" 

.github/workflows/test-suite.yml

@@ -159,6 +159,28 @@ jobs:
     - name: Show cache stats
       if: env.SELF_HOSTED_RUNNERS == 'true'
       run: sccache --show-stats
+  http-api-tests:
+    name: http-api-tests
+    needs: [check-labels]
+    if: needs.check-labels.outputs.skip_ci != 'true'
+    # Use self-hosted runners only on the sigp repo.
+    runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "large"]') || 'ubuntu-latest'  }}
+    env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+    - uses: actions/checkout@v4
+    - name: Get latest version of stable Rust
+      if: env.SELF_HOSTED_RUNNERS == 'false'
+      uses: moonrepo/setup-rust@v1
+      with:
+          channel: stable
+          cache-target: release
+          bins: cargo-nextest
+    - name: Run http_api tests for all recent forks
+      run: make test-http-api
+    - name: Show cache stats
+      if: env.SELF_HOSTED_RUNNERS == 'true'
+      run: sccache --show-stats
   op-pool-tests:
     name: op-pool-tests
     needs: [check-labels]
@@ -475,6 +497,7 @@ jobs:
       'op-pool-tests',
       'network-tests',
       'slasher-tests',
+      'http-api-tests',
       'debug-tests-ubuntu',
       'state-transition-vectors-ubuntu',
       'ef-tests-ubuntu',