feat: restore edge/block collection and runtime consumption (#10869)

* feat: restore edge/block PGO collection and runtime consumption

Re-enable DOTNET_WritePGOData in Dockerfile.pgo to collect JIT edge/block
profiling data during PGO runs. Add PgoTrim tool to strip cold methods
and compress the ~31MB .jit file to ~438KB gzip. The weekly workflow now
trims and commits nethermind.jit.gz alongside the .mibc profile.

The production Dockerfile decompresses the .jit.gz at build time, and
entrypoint.sh enables DOTNET_ReadPGOData when the file exists, giving
the JIT branch layout hints and guarded devirtualization data.

* chore(pgo): update PGO profile (#10870)

chore(pgo): update PGO profile from weekly collection

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* chore(pgo): update PGO profile (#10870)

chore(pgo): update PGO profile from weekly collection

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fix: address PGO pipeline review feedback

- Flush pending histogram count at method boundaries in PgoTrim to
  prevent silent data loss if kind-177 has no paired 195/196
- Use [ -s ] instead of [ -f ] in Dockerfile so empty .jit.gz placeholder
  does not fail gzip decompression
- Log when edge/block PGO is enabled in entrypoint.sh for operator
  visibility
- Update Dockerfile.pgo header comment to mention both .nettrace and
  .jit outputs

* fix: use proper nullable tuple pattern in PgoTrim

Replace `is var (x, y)` (always matches, even when null) with
`is (string x, string y)` so pending histogram entries are only
emitted when actually present.

* chore: default reproducible benchmarks to flat db layout

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: benaadams

.github/workflows/collect-pgo-profile.yml

@@ -247,6 +247,8 @@ jobs:
 
           echo "=== .nettrace files (EventPipe method traces) ==="
           find "${RUNNER_TEMP}/pgo-data" -name "*.nettrace" -type f -ls 2>/dev/null || true
+          echo "=== .jit files (edge/block profiling data) ==="
+          find "${RUNNER_TEMP}/pgo-data" -name "*.jit" -type f -ls 2>/dev/null || true
 
       - name: Set up .NET
         if: always()
@@ -334,6 +336,20 @@ jobs:
           ls -lh "${mibc_file}"
           echo "mibc_file=${mibc_file}" >> "${GITHUB_OUTPUT}"
 
+          # Step 2: Trim .jit edge/block profiling data for runtime PGO
+          # .jit files (from DOTNET_WritePGOData) use a JIT-internal text format read
+          # back via DOTNET_ReadPGOData — they can't be merged into .mibc by dotnet-pgo.
+          jit_file=$(find "${pgo_dir}" -name "*.jit" -type f -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -1 | cut -d' ' -f2- || true)
+          jit_gz="${RUNNER_TEMP}/nethermind.jit.gz"
+          if [[ -n "${jit_file}" ]]; then
+            echo "Trimming ${jit_file} with PgoTrim..."
+            dotnet run --project "${GITHUB_WORKSPACE}/tools/PgoTrim" -c Release -- \
+              "${jit_file}" "${jit_gz}" --min-block 100 --min-edge 250
+            echo "jit_gz=${jit_gz}" >> "${GITHUB_OUTPUT}"
+          else
+            echo "No .jit file found — skipping edge profiling trim."
+          fi
+
       - name: Upload .mibc artifact
         if: always()
         uses: actions/upload-artifact@v7
@@ -342,6 +358,14 @@ jobs:
           path: ${{ steps.convert.outputs.mibc_file }}
           retention-days: 90
 
+      - name: Upload trimmed .jit.gz artifact
+        if: always() && steps.convert.outputs.jit_gz != ''
+        uses: actions/upload-artifact@v7
+        with:
+          name: nethermind-pgo-jit
+          path: ${{ steps.convert.outputs.jit_gz }}
+          retention-days: 90
+
       - name: Upload raw PGO data
         if: always()
         uses: actions/upload-artifact@v7
@@ -366,10 +390,17 @@ jobs:
           name: nethermind-pgo-profile
           path: src/Nethermind/Nethermind.Runner/pgo/
 
+      - name: Download .jit.gz artifact
+        uses: actions/download-artifact@v7
+        continue-on-error: true
+        with:
+          name: nethermind-pgo-jit
+          path: src/Nethermind/Nethermind.Runner/pgo/
+
       - name: Check for changes
         id: diff
         run: |
-          if git diff --quiet src/Nethermind/Nethermind.Runner/pgo/nethermind.mibc; then
+          if git diff --quiet src/Nethermind/Nethermind.Runner/pgo/; then
             echo "changed=false" >> "${GITHUB_OUTPUT}"
             echo "No changes to PGO profile."
           else
@@ -387,13 +418,15 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git checkout -b "${branch}"
           git add src/Nethermind/Nethermind.Runner/pgo/nethermind.mibc
+          git add src/Nethermind/Nethermind.Runner/pgo/nethermind.jit.gz 2>/dev/null || true
           git commit -m "chore(pgo): update PGO profile from weekly collection"
           git push -u origin "${branch}"
           gh pr create \
             --title "chore(pgo): update PGO profile" \
             --body "$(cat <<'EOF'
           ## Summary
           - Weekly automated update of the PGO (.mibc) profile used for R2R compilation
+          - Updated edge/block profiling data (.jit.gz) for runtime branch optimization
           - Generated from 5000 mainnet blocks via EventPipe profiling
 
           ## Test plan

.github/workflows/run-expb-reproducible-benchmarks.yml

@@ -18,7 +18,7 @@ on:
         options:
           - halfpath
           - flat
-        default: halfpath
+        default: flat
       payload_set:
         description: Payload set mode
         required: true
@@ -168,7 +168,7 @@ jobs:
               should_run="false"
             fi
             branch="${PR_HEAD_BRANCH}"
-            state_layout="halfpath"
+            state_layout="flat"
             payload_set="superblocks"
             payload_sets='["superblocks","realblocks"]'
             expb_repo="NethermindEth/execution-payloads-benchmarks"
@@ -181,7 +181,7 @@ jobs:
             run_count="1"
           else
             branch="${PUSH_BRANCH}"
-            state_layout="halfpath"
+            state_layout="flat"
             payload_set="superblocks"
             payload_sets='["superblocks","realblocks"]'
             expb_repo="NethermindEth/execution-payloads-benchmarks"

Dockerfile

@@ -30,6 +30,15 @@ FROM mcr.microsoft.com/dotnet/aspnet:10.0.5-noble@sha256:a04d1c1d2d2611904949405
 
 WORKDIR /nethermind
 
+# Decompress edge/block PGO data for runtime JIT optimization
+# The .jit.gz is generated by the weekly PGO collection workflow
+COPY src/Nethermind/Nethermind.Runner/pgo/ /tmp/pgo/
+RUN mkdir -p pgo && \
+  if [ -s /tmp/pgo/nethermind.jit.gz ]; then \
+    gzip -d -c /tmp/pgo/nethermind.jit.gz > pgo/nethermind.jit; \
+  fi && \
+  rm -rf /tmp/pgo
+
 VOLUME /nethermind/keystore
 VOLUME /nethermind/logs
 VOLUME /nethermind/nethermind_db

Dockerfile.pgo

@@ -4,6 +4,8 @@
 # Builds Nethermind WITHOUT ReadyToRun so the JIT compiles all methods,
 # then enables EventPipe to collect profile data for Static PGO.
 # The resulting .nettrace file is converted to .mibc offline.
+# Also writes edge/block profiling data (.jit) via DOTNET_WritePGOData
+# which is trimmed by PgoTrim and consumed at runtime via DOTNET_ReadPGOData.
 
 FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:10.0.201-noble@sha256:478b9038d187e5b5c29bfa8173ded5d29e864b5ad06102a12106380ee01e2e49 AS build
 
@@ -47,6 +49,10 @@ ENV DOTNET_EnableEventPipe=1
 ENV DOTNET_EventPipeCircularMB=256
 ENV DOTNET_EventPipeOutputPath=/nethermind/pgo/nethermind-{pid}.nettrace
 ENV DOTNET_EventPipeConfig=Microsoft-Windows-DotNETRuntime:0x1E000080018:4
+# Write edge/block profiling data after Tier-1 compilation for branch optimization
+ENV DOTNET_TieredPGO=1
+ENV DOTNET_WritePGOData=1
+ENV DOTNET_PGODataPath=/nethermind/pgo/nethermind.jit
 
 VOLUME /nethermind/keystore
 VOLUME /nethermind/logs

scripts/entrypoint.sh

@@ -66,4 +66,11 @@ if [[ -z "${DOTNET_GCLargePages:-}" ]]; then
   fi
 fi
 
+# Enable edge/block PGO if the trimmed profile exists
+if [[ -z "${DOTNET_ReadPGOData:-}" ]] && [[ -f "/nethermind/pgo/nethermind.jit" ]]; then
+  export DOTNET_ReadPGOData=1
+  export DOTNET_PGODataPath=/nethermind/pgo/nethermind.jit
+  echo "Edge/block PGO enabled: ${DOTNET_PGODataPath}"
+fi
+
 exec ./nethermind "$@"

src/Nethermind/Nethermind.Runner/pgo/nethermind.jit.gz

@@ -0,0 +1,5 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+</Project>