Merge main-v0.14.1-committer into main

.cargo/config.toml

@@ -2,11 +2,3 @@
 LLVM_SYS_191_PREFIX = "/usr/lib/llvm-19/"
 MLIR_SYS_190_PREFIX = "/usr/lib/llvm-19/"
 TABLEGEN_190_PREFIX = "/usr/lib/llvm-19/"
-
-# Use `lld` for linking instead of `ld`, since we run out of memory while linking with `ld` on
-# 16-cores linux machines, see:
-# https://nnethercote.github.io/perf-book/build-configuration.html#linking.
-# TODO(Gilad): remove this once `rust` stabilizes `lld` as the default linker, currently only on nightly:
-# https://github.com/rust-lang/rust/issues/39915#issuecomment-618726211
-[target.'cfg(all(target_os = "linux"))']
-rustflags = ["-Clink-arg=-fuse-ld=lld"]

.dockerignore

@@ -3,4 +3,3 @@
 /logs
 **/target/
 .git
-!crates/papyrus_load_test/resources

.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# GitHub CODEOWNERS file
+# This file defines who must review and approve changes to specific files or directories
+# For more information, see: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
+
+# Require approval from the sequencer-cargo-owners team for workspace dependency changes
+Cargo.toml @starkware-libs/sequencer-cargo-owners

.github/ISSUE_TEMPLATE/01_BUG_REPORT.md

@@ -1,14 +1,14 @@
 ---
 name: Bug Report
-about: Create a report to help Papyrus to improve
+about: Create a report to help Apollo to improve
 title: "bug: "
 labels: "bug"
 assignees: ""
 ---
 
 # Bug Report
 
-**Papyrus version:**
+**Apollo version:**
 
 <!-- Please specify commit or tag version. -->
 

.github/actions/bootstrap/action.yml

@@ -12,14 +12,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install ldd
-      shell: bash
-      run: |
-        sudo apt-get update
-        sudo apt-get remove -y man-db
-        sudo apt-mark hold man-db
-        sudo apt-get install -y --no-install-recommends build-essential binutils lld
-
     - name: Install rust.
       uses: ./.github/actions/install_rust
       with:

.github/actions/install_rust/action.yml

@@ -21,7 +21,6 @@ runs:
         # Install additional non-default toolchains (for rustfmt for example), NOP if input omitted.
         channel: ${{ inputs.extra_rust_toolchains }}
       env:
-        RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
         GITHUB_TOKEN: ${{ inputs.github_token }}
 
     # This installation is _not_ cached, but takes a couple seconds: it's downloading prepackaged

.github/workflows/blockifier_ci.yml

@@ -37,7 +37,9 @@ on:
       - "scripts/sequencer-ci.Dockerfile"
 
 env:
-  RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=lld"
+  RUSTFLAGS: "-D warnings"
+  TRANSFERS_BENCHMARK_CAIRO_NATIVE_TIME_LIMIT_NS: "200000000"
+  TRANSFERS_BENCHMARK_VM_TIME_LIMIT_NS: "1000000000"
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 # Also, create different concurrency groups for different pushed commits, on push events.
@@ -92,3 +94,41 @@ jobs:
       # tracing is not activated by any workspace crate; test the build.
       - run: cargo build -p blockifier --features tracing
       - run: cargo test -p blockifier --features tracing
+
+  benchmarking:
+    runs-on: namespace-profile-medium-ubuntu-24-04-amd64
+    if: ${{ github.event_name == 'pull_request' }}
+    steps:
+      # Checkout the base branch to benchmark the old code.
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+
+      - uses: ./.github/actions/bootstrap
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      # Restore cargo artifacts build cache.
+      - name: Restore Cargo build cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: blockifier-bench
+          cache-on-failure: true
+          cache-workspace-crates: true
+
+      # Benchmark the base branch code.
+      - run: cargo run -p bench_tools -- run --package blockifier --out /tmp/base_results
+
+      # Checkout the current branch to benchmark the new code.
+      - uses: actions/checkout@v4
+        with:
+          clean: false
+
+      # Benchmark the current branch and compare to the previous run.
+      - run: |
+          cargo run -p bench_tools -- run-and-compare \
+            --package blockifier \
+            --out /tmp/new_results \
+            --regression-limit 8.0 \
+            --set-absolute-time-ns-limit transfers_benchmark_cairo_native ${TRANSFERS_BENCHMARK_CAIRO_NATIVE_TIME_LIMIT_NS} \
+            --set-absolute-time-ns-limit transfers_benchmark_vm ${TRANSFERS_BENCHMARK_VM_TIME_LIMIT_NS}

.github/workflows/blockifier_compiled_cairo.yml

@@ -14,7 +14,7 @@ on:
       - "scripts/dependencies.sh"
 
 env:
-  RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=lld"
+  RUSTFLAGS: "-D warnings"
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 # Also, create different concurrency groups for different pushed commits, on push events.

.github/workflows/blockifier_reexecution_ci.yml

@@ -19,7 +19,7 @@ on:
       - "scripts/sequencer-ci.Dockerfile"
 
 env:
-  RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=lld"
+  RUSTFLAGS: "-D warnings"
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 # Also, create different concurrency groups for different pushed commits, on push events.

.github/workflows/committer_and_os_cli_push.yml

@@ -30,7 +30,7 @@ on:
       - "scripts/dependencies.sh"
 
 env:
-  RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=lld"
+  RUSTFLAGS: "-D warnings"
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 # Also, create different concurrency groups for different pushed commits, on push events.

.github/workflows/committer_ci.yml

@@ -18,7 +18,9 @@ on:
       - "scripts/dependencies.sh"
 
 env:
-  RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=lld"
+  RUSTFLAGS: "-D warnings"
+  FULL_COMMITTER_FLOW_TIME_LIMIT_NS: "25000000"
+  TREE_COMPUTATION_FLOW_TIME_LIMIT_NS: "20000000"
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 # Also, create different concurrency groups for different pushed commits, on push events.
@@ -62,113 +64,61 @@ jobs:
       - run: gcloud storage cp -r gs://committer-testing-artifacts/$BENCH_INPUT_FILES_PREFIX/* ./crates/starknet_committer_and_os_cli/test_inputs
       - run: cargo test -p starknet_committer_and_os_cli --release -- --include-ignored test_regression
 
-  benchmarking:
-    runs-on: namespace-profile-medium-ubuntu-24-04-amd64
-    if: ${{ github.event_name == 'pull_request' }}
-    steps:
-      # Checkout the base branch to get the old code.
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.base_ref }}
-
-      # Setup pypy and link to the location expected by .cargo/config.toml.
-      # Python + requirements are needed to compile the OS.
-      - uses: actions/setup-python@v5
-        id: setup-pypy
-        with:
-          python-version: "pypy3.9"
-          cache: "pip"
-      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
-      - env:
-          LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
-        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-      - run: pip install -r scripts/requirements.txt
-
-      - uses: ./.github/actions/bootstrap
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-
-      # Download the old benchmark inputs.
-      - id: auth
-        uses: "google-github-actions/auth@v2"
-        with:
-          credentials_json: ${{ secrets.COMMITER_PRODUCTS_EXT_WRITER_JSON }}
-      - uses: "google-github-actions/setup-gcloud@v2"
-      - run: echo "OLD_BENCH_INPUT_FILES_PREFIX=$(cat ./crates/starknet_committer_and_os_cli/src/committer_cli/tests/flow_test_files_prefix)" >> $GITHUB_ENV
-      - run: gcloud storage cp -r gs://committer-testing-artifacts/$OLD_BENCH_INPUT_FILES_PREFIX/* ./crates/starknet_committer_and_os_cli/test_inputs
-
-      # List the existing benchmarks.
-      - run: |
-          cargo bench -p starknet_committer_and_os_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
-
-      # Benchmark the old code.
-      - run: cargo bench -p starknet_committer_and_os_cli
-
-      # Backup the downloaded files to avoid re-downloading them if they didn't change (overwritten by checkout).
-      - run: mv ./crates/starknet_committer_and_os_cli/test_inputs/tree_flow_inputs.json ./crates/starknet_committer_and_os_cli/test_inputs/tree_flow_inputs.json_bu
-      - run: mv ./crates/starknet_committer_and_os_cli/test_inputs/committer_flow_inputs.json ./crates/starknet_committer_and_os_cli/test_inputs/committer_flow_inputs.json_bu
-
-      # Checkout the new code.
-      - uses: actions/checkout@v4
-        with:
-          clean: false
-      - run: echo "NEW_BENCH_INPUT_FILES_PREFIX=$(cat ./crates/starknet_committer_and_os_cli/src/committer_cli/tests/flow_test_files_prefix)" >> $GITHUB_ENV
-
-      # If the pip requirements on the old commit are different from the new commit, re-install them.
-      - run: pip install -r scripts/requirements.txt
-
-      # Input files didn't change.
-      - if: env.OLD_BENCH_INPUT_FILES_PREFIX == env.NEW_BENCH_INPUT_FILES_PREFIX
-        run: |
-          mv ./crates/starknet_committer_and_os_cli/test_inputs/tree_flow_inputs.json_bu ./crates/starknet_committer_and_os_cli/test_inputs/tree_flow_inputs.json
-          mv ./crates/starknet_committer_and_os_cli/test_inputs/committer_flow_inputs.json_bu ./crates/starknet_committer_and_os_cli/test_inputs/committer_flow_inputs.json
-
-      # Input files did change, download new inputs.
-      - if: env.OLD_BENCH_INPUT_FILES_PREFIX != env.NEW_BENCH_INPUT_FILES_PREFIX
-        run: |
-          gcloud storage cp -r gs://committer-testing-artifacts/$NEW_BENCH_INPUT_FILES_PREFIX/* ./crates/starknet_committer_and_os_cli/test_inputs
-
-      # Benchmark the new code, splitting the benchmarks, and prepare the results for posting a comment.
-      - run: bash ./crates/starknet_committer_and_os_cli/benches/bench_split_and_prepare_post.sh benchmarks_list.txt benchmark_comment_body.txt
-
-      - run: echo BENCHES_RESULT=$(cat benchmark_comment_body.txt) >> $GITHUB_ENV
-
-      # Comment with a link to the workflow (or update existing comment on rerun).
-      - name: Find Comment
-        if: github.event_name == 'pull_request'
-        uses: starkware-libs/find-comment@v3
-        id: find-benchmark-comment
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: "github-actions[bot]"
-          body-includes: Benchmark movements
-
-      - name: Create comment
-        # If the PR number is found and the comment is not found, create a new comment.
-        if: github.event_name == 'pull_request'
-          && steps.find-benchmark-comment.outputs.comment-id == ''
-        uses: starkware-libs/create-or-update-comment@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body: ${{ env.BENCHES_RESULT }}
-
-      - name: Update comment
-        # If the PR number is found and the comment exists, update it.
-        if: github.event_name == 'pull_request'
-          && steps.find-benchmark-comment.outputs.comment-id != ''
-        uses: starkware-libs/create-or-update-comment@v4
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          comment-id: ${{ steps.find-benchmark-comment.outputs.comment-id }}
-          edit-mode: replace
-          body: ${{ env.BENCHES_RESULT }}
-
-      # Fail the job if benchmarks exceeded the threshold (after posting the comment).
-      - name: Check benchmark results
-        run: |
-          if grep -q "CI WILL FAIL" benchmark_comment_body.txt; then
-            echo "Benchmarks exceeded the regression threshold. See the PR comment for details."
-            exit 1
-          fi
+  # TODO(Aviv): Uncomment.
+  # benchmarking:
+  #   runs-on: namespace-profile-medium-ubuntu-24-04-amd64
+  #   if: ${{ github.event_name == 'pull_request' }}
+  #   steps:
+  #     # Checkout the base branch to benchmark the old code.
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         ref: ${{ github.base_ref }}
+
+  #     # Setup pypy and link to the location expected by .cargo/config.toml.
+  #     # Python + requirements are needed to compile the OS.
+  #     - uses: actions/setup-python@v5
+  #       id: setup-pypy
+  #       with:
+  #         python-version: "pypy3.9"
+  #         cache: "pip"
+  #     - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+  #     - env:
+  #         LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
+  #       run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+  #     - run: pip install -r scripts/requirements.txt
+
+  #     - uses: ./.github/actions/bootstrap
+  #       with:
+  #         github_token: ${{ secrets.GITHUB_TOKEN }}
+
+  #     # Auth with GCS for bench_tools to download inputs.
+  #     - id: auth
+  #       uses: "google-github-actions/auth@v2"
+  #       with:
+  #         credentials_json: ${{ secrets.COMMITER_PRODUCTS_EXT_WRITER_JSON }}
+  #     - uses: "google-github-actions/setup-gcloud@v2"
+
+  #     # Benchmark the base branch code.
+  #     - run: cargo run -p bench_tools -- run --package starknet_committer_and_os_cli --out /tmp/base_results
+
+  #     # Save the input files from the base branch to use in the second benchmark.
+  #     - run: cp -r crates/starknet_committer_and_os_cli/test_inputs /tmp/test_inputs
+
+  #     # Checkout the current branch to benchmark the new code.
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         clean: false
+
+  #     # Re-install pip requirements in case they changed between branches.
+  #     - run: pip install -r scripts/requirements.txt
+
+  #     # Benchmark the current branch and compare to the previous run.
+  #     # Use the saved inputs and enforce absolute time limits (full_committer_flow: 25ms, tree_computation_flow: 20ms).
+  #     - run: |
+  #         cargo run -p bench_tools -- run-and-compare \
+  #           --package starknet_committer_and_os_cli \
+  #           --out /tmp/new_results \
+  #           --regression-limit 8.0 \
+  #           --input-dir /tmp/test_inputs \
+  #           --set-absolute-time-ns-limit full_committer_flow ${FULL_COMMITTER_FLOW_TIME_LIMIT_NS} \
+  #           --set-absolute-time-ns-limit tree_computation_flow ${TREE_COMPUTATION_FLOW_TIME_LIMIT_NS}

.github/workflows/hybrid_system_test.yaml

@@ -18,6 +18,7 @@ env:
   cluster_name: hybrid-system-test
   crate_triggers: "apollo_node,apollo_deployments,apollo_integration_tests"
   path_triggers: ".github/workflows/hybrid_system_test.yaml,scripts/*.py,scripts/system_tests/**/*.py,deployments/sequencer/**"
+  path_triggers_exclude: "scripts/prod/**/*"
   pvc_storage_class_name: "premium-rwo"
   anvil_port: "8545"
   dockerfile_base_path: ./deployments/images/sequencer
@@ -61,7 +62,8 @@ jobs:
           python ./scripts/check_test_trigger.py --output_file $OUTPUT_FILE \
             --commit_id ${{ github.event.pull_request.base.sha }} \
             --crate_triggers ${{ env.crate_triggers }} \
-            --path_triggers ${{ env.path_triggers }}
+            --path_triggers ${{ env.path_triggers }} \
+            --path_triggers_exclude ${{ env.path_triggers_exclude }}
 
           should_run=$(cat "$OUTPUT_FILE")
           echo "Captured output: $should_run"
@@ -298,7 +300,7 @@ jobs:
         run: chmod +x ./target/debug/sequencer_node_setup ./target/debug/sequencer_simulator
 
       - name: Create storage files
-        run: ./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-consolidated 1 --n-distributed 0
+        run: ./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-consolidated 1 --n-hybrid 0 --n-distributed 0
 
       - name: Export application config dir
         run: |
@@ -363,7 +365,7 @@ jobs:
             --interval ${{ env.check_interval_sec }}
 
       - name: Copy state and restart pod
-        run: pipenv run python ./scripts/system_tests/copy_state_and_restart.py --deployment_config_path ${{ env.deployment_config_path }} --data-dir "./output/data/node_0/executable_0"
+        run: pipenv run python ./scripts/system_tests/copy_state_and_restart.py --deployment_config_path ${{ env.deployment_config_path }} --data-dir "./output/data/node_0"
 
       - name: Port-forward Anvil pod to localhost:${{ env.anvil_port }}
         run: |