diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0b4d57e9c..7760774b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,454 +16,495 @@ concurrency: cancel-in-progress: true jobs: - ledger-tests: - runs-on: ubuntu-20.04 - steps: - - name: Git checkout - uses: actions/checkout@v4 - - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - name: Setup Rust - run: | - # Nightly to be able to use `--report-time` below - rustup install nightly - rustup override set nightly - - name: Download circuits files - run: | - git clone --depth 1 https://github.com/openmina/circuit-blobs.git - ln -s -b $PWD/circuit-blobs/* ledger/ - - name: Build ledger tests - run: | - cd ledger - cargo build --release --tests - - name: Run ledger tests - run: | - cd ledger - cargo test --release -- -Z unstable-options --report-time - - p2p-tests: - runs-on: ubuntu-20.04 - steps: - - name: Git checkout - uses: actions/checkout@v4 - - - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup Rust - run: | - rustup default 1.80 - rustup component add rustfmt - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" - - - name: Test p2p crate - run: | - cargo test -p p2p --tests - - - build: - runs-on: ubuntu-20.04 - steps: - - name: Git checkout - uses: actions/checkout@v4 - - - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup Rust - run: | - rustup default 1.80 - rustup component add rustfmt - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" - - - name: Release build - run: | - cargo build --release --bin openmina - - - name: Upload binaries - uses: actions/upload-artifact@v4 - with: - name: bin - path: target/release/openmina - - build_wasm: - runs-on: ubuntu-20.04 - steps: - - name: Git checkout - uses: actions/checkout@v4 - - - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup Rust - run: | - rustup default nightly - rustup component add rustfmt rust-src - rustup target add wasm32-unknown-unknown - cargo install -f wasm-bindgen-cli --version 0.2.93 - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" - - - name: Release build - run: | - cd node/web - cargo +nightly build --release --target wasm32-unknown-unknown - wasm-bindgen --keep-debug --web --out-dir pkg ../../target/wasm32-unknown-unknown/release/openmina_node_web.wasm - - build-tests: - runs-on: ubuntu-20.04 - steps: - - name: Git checkout - uses: actions/checkout@v4 - - - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup Rust - run: | - rustup default 1.80 - rustup component add rustfmt - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" - - - name: Build tests - run: | - mkdir -p target/release/tests - - cargo build --release --tests --package=openmina-node-testing --package=cli - cargo build --release --tests --package=openmina-node-testing --package=cli --message-format=json > cargo-build-test.json - jq -r '. | select(.executable != null and (.target.kind | (contains(["test"])))) | [.target.name, .executable ] | @tsv' cargo-build-test.json > tests.tsv - while read NAME FILE; do cp -a $FILE target/release/tests/$NAME; done < tests.tsv - - - name: Upload tests - uses: actions/upload-artifact@v4 - with: - name: tests - path: target/release/tests - - build-tests-webrtc: - runs-on: ubuntu-20.04 - steps: - - name: Git checkout - uses: actions/checkout@v4 - - - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup Rust - run: | - rustup default 1.80 - rustup component add rustfmt - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" - - - name: Build tests - run: | - mkdir -p target/release/tests - - cargo build --release --features=scenario-generators,p2p-webrtc --package=openmina-node-testing --tests - cargo build --release --features=scenario-generators,p2p-webrtc --package=openmina-node-testing --tests --message-format=json > cargo-build-test.json - jq -r '. | select(.executable != null and (.target.kind | (contains(["test"])))) | [.target.name, .executable ] | @tsv' cargo-build-test.json > tests.tsv - while read NAME FILE; do cp -a $FILE target/release/tests/webrtc_$NAME; done < tests.tsv - - - name: Upload tests - uses: actions/upload-artifact@v4 - with: - name: tests-webrtc - path: target/release/tests - - p2p-scenario-tests: - needs: [ build-tests, build-tests-webrtc ] - runs-on: ubuntu-20.04 - container: - image: minaprotocol/mina-daemon:3.0.0-dc6bf78-focal-devnet - options: --volume debugger_data:/tmp/db - env: - BPF_ALIAS: /coda/0.0.1/29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6-0.0.0.0 + # ledger-tests: + # runs-on: ubuntu-20.04 + # steps: + # - name: Git checkout + # uses: actions/checkout@v4 + # - name: Setup build dependencies + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler + # - name: Setup Rust + # run: | + # # Nightly to be able to use `--report-time` below + # rustup install nightly + # rustup override set nightly + # - name: Download circuits files + # run: | + # git clone --depth 1 https://github.com/openmina/circuit-blobs.git + # ln -s -b $PWD/circuit-blobs/* ledger/ + # - name: Build ledger tests + # run: | + # cd ledger + # cargo build --release --tests + # - name: Run ledger tests + # run: | + # cd ledger + # cargo test --release -- -Z unstable-options --report-time + + ledger-wasm-tests: strategy: - matrix: - test: [p2p_basic_connections, p2p_basic_incoming, p2p_basic_outgoing, p2p_pubsub, p2p_kad, - webrtc_p2p_basic_connections] fail-fast: false - - services: - network-debugger: - image: openmina/mina-network-debugger:23385c61 - options: --privileged --init --volume debugger_data:/tmp/db --volume /sys/kernel/debug:/sys/kernel/debug - env: - SERVER_PORT: 80 - FIREWALL_INTERFACE: lo - RUST_LOG: info - DB_PATH: /tmp/db - ports: - - 80:80 - - steps: - - name: Download tests - uses: actions/download-artifact@v4 - with: - pattern: tests* - merge-multiple: true - - - name: Setup permissions - run: | - chmod +x ./${{ matrix.test }} - - # TODO: use curl - - name: Wait for the debugger - run: | - sleep 5 - - - name: Run the test - run: | - ./${{ matrix.test }} --test-threads=1 - - - name: Archive network debugger database - uses: actions/upload-artifact@v4 - with: - name: network-debugger-${{ matrix.test }} - path: /tmp/db - if: ${{ always() }} - - k8s-peers: - runs-on: ubuntu-20.04 - # TODO: query cluster for actual addresses, or specify then on deployment - env: - PEERS: | - /ip4/135.181.217.23/tcp/31881/p2p/12D3KooWS4TMSjrAS4Cj31PgjZ9KgeHh5goLP65M5GSriF28d7Jx - /ip4/135.181.217.23/tcp/30386/p2p/12D3KooWK92cYz26JqBE9vM9s9Jd9pJKcNxLd7VVRupU7YG5NupU - /ip4/135.181.217.23/tcp/32272/p2p/12D3KooWSU1DYZYVA7wAYLvLsH6yVS8oV1sMJUcC7VtUxCgtsWkJ - /ip4/135.181.217.23/tcp/32218/p2p/12D3KooWM5m9QqHpDkPJi54GPP6rGFpzo7E274husrModPSLQ7tn - /ip4/135.181.217.23/tcp/30798/p2p/12D3KooWCk2QSmQH2XbtpDXiSPUq6wb2LB2JaExHRXkJggmEfN4J - /ip4/135.181.217.23/tcp/31631/p2p/12D3KooWQ1642Dzm57Kr8tmTwS9NRFaJPy4ysaQ2ne3ZYwQn5qCk - /ip4/135.181.217.23/tcp/30196/p2p/12D3KooWHK67syE2LeTz5EnNqCe5ZFf9SoZRFN4AdHVZsL31WkMn - /ip4/135.181.217.23/tcp/30790/p2p/12D3KooWDwxrG5u12FzXAFyK7vd8aHnEQf4dwoboBJ72FUS179xK - /ip4/135.181.217.23/tcp/30070/p2p/12D3KooWEowA3VakSddUjZuBTK3HJhNM7sRqwWDbqtPtwymAMCcy - outputs: - peers: ${{ steps.peers.outputs.peers }} - steps: - - id: peers - name: Fetch k8s OCaml peers - run: | - { - echo 'peers<> "$GITHUB_OUTPUT" - - scenario-tests: - needs: - - k8s-peers - - build-tests - # - build-tests-webrtc - runs-on: ubuntu-20.04 - container: - image: minaprotocol/mina-daemon:3.0.0-dc6bf78-focal-devnet - options: --volume debugger_data:/tmp/db - env: - # to allow local addrs discovery - OPENMINA_DISCOVERY_FILTER_ADDR: false - # to allow connection with replayer - # TODO: remove when replayer supports identify - KEEP_CONNECTION_WITH_UNKNOWN_STREAM: true - OPENMINA_SCENARIO_SEEDS: ${{ needs.k8s-peers.outputs.peers }} - BPF_ALIAS: /coda/0.0.1/29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6-0.0.0.0 - strategy: matrix: - test: - - single_node - - multi_node_initial_joining - - multi_node_peer_discovery - - multi_node_propagate_block - - connection_discovery_ocaml_to_rust_via_seed - - connection_discovery_ocaml_to_rust - - connection_discovery_rust_as_seed - - connection_discovery_rust_to_ocaml_via_seed - - connection_discovery_rust_to_ocaml - # - webrtc_single_node - # - webrtc_multi_node - fail-fast: false - - services: - network-debugger: - image: openmina/mina-network-debugger:23385c61 - options: --privileged --init --volume debugger_data:/tmp/db --volume /sys/kernel/debug:/sys/kernel/debug - env: - SERVER_PORT: 80 - FIREWALL_INTERFACE: lo - RUST_LOG: info - DB_PATH: /tmp/db - ports: - - 80:80 - - steps: - - name: Download tests - uses: actions/download-artifact@v4 - with: - pattern: tests* - merge-multiple: true - - - name: Setup permissions - run: | - chmod +x ./${{ matrix.test }} - - # TODO: use curl - - name: Wait for the debugger - run: | - sleep 5 - - - name: Run the test - run: | - ./${{ matrix.test }} --test-threads=1 - - - name: Archive network debugger database - uses: actions/upload-artifact@v4 - with: - name: network-debugger-${{ matrix.test }} - path: /tmp/db - if: ${{ always() }} - - record-replay-tests: - needs: - - k8s-peers - - build-tests - - build-tests-webrtc - runs-on: ubuntu-20.04 - container: - image: minaprotocol/mina-daemon:3.0.0-dc6bf78-focal-devnet + driver: + ## FIXME: Activate chrome, but now it timeouts when tests run more than 10 minutes + ## Timeout example: https://github.com/openmina/ledger/actions/runs/5245920744/jobs/9474043794 + # - --chrome + - --firefox + # - --node -- --features in_nodejs -Z build-std=std,panic_abort + runs-on: ubuntu-latest env: - # to allow local addrs discovery - OPENMINA_DISCOVERY_FILTER_ADDR: false - OPENMINA_SCENARIO_SEEDS: ${{ needs.k8s-peers.outputs.peers }} - strategy: - matrix: - test: [record_replay, webrtc_record_replay] - fail-fast: false - + RUSTFLAGS: "-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--max-memory=4294967296" + RUST_LOG: wasm_bindgen_test_runner + RUST_BACKTRACE: full + CARGO_PROFILE_RELEASE_OVERFLOW_CHECKS: true + CARGO_PROFILE_RELEASE_DEBUG_ASSERTIONS: true + CARGO_PROFILE_RELEASE_DEBUG: true steps: - - name: Download tests - uses: actions/download-artifact@v4 - with: - pattern: tests* - merge-multiple: true - - - name: Setup permissions - run: | - chmod +x ./${{ matrix.test }} + - uses: actions/checkout@v3 - - name: Run the test - run: | - ./${{ matrix.test }} --test-threads=1 - - - bootstrap-test: - needs: [ k8s-peers, build, build-tests ] - runs-on: ubuntu-20.04 - env: - PEERS_LIST: ${{ needs.k8s-peers.outputs.peers }} - PEER_LIST_FILE: peer-list.txt - WORK_DIR: data - BPF_ALIAS: /coda/0.0.1/29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6-0.0.0.0 - - services: - network-debugger: - image: openmina/mina-network-debugger:23385c61 - options: --privileged --init --volume /tmp/db:/tmp/db --volume /sys/kernel/debug:/sys/kernel/debug - env: - SERVER_PORT: 80 - FIREWALL_INTERFACE: lo - RUST_LOG: info - DB_PATH: /tmp/db - ports: - - 80:80 - - steps: - - name: Download binary - uses: actions/download-artifact@v4 + - name: Install toolchain + uses: actions-rs/toolchain@v1 with: - name: bin + toolchain: nightly + override: true + profile: minimal + components: rust-src - - name: Download test - uses: actions/download-artifact@v4 - with: - pattern: tests* - merge-multiple: true - - - name: Fix permissions - run: | - chmod +x bootstrap openmina - - # TODO: use curl - - name: Wait for the debugger - run: | - sleep 5 - - - name: Peer List File - run: | - for PEER in $PEERS_LIST; do echo $PEER; done > $PEER_LIST_FILE - cat $PEER_LIST_FILE - - - name: Bootstrap node - env: - OPENMINA_COMMAND: openmina - NO_PEER_DISCOVERY: "true" - OUT_PATH: ${{ env.WORK_DIR }}/logs/bootstrap_output - RECORD: state-with-input-actions + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Download circuits files run: | - mkdir -p $OUT_PATH - PATH=$PATH:$(pwd) OPENMINA_COMMAND=openmina NO_PEER_DISCOVERY=true ./bootstrap --nocapture || { - echo "::group::Stderr" - cat $OUT_PATH.stderr - echo "::endgroup::" - exit 1 - } - - - name: Upload logs - uses: actions/upload-artifact@v4 - with: - name: bootstrap-logs - path: ${{ env.WORK_DIR }}/logs/* - if: ${{ failure() }} - - - name: Upload record - uses: actions/upload-artifact@v4 - with: - name: bootstrap-record - path: ${{ env.WORK_DIR }}/recorder/* - if: ${{ failure() }} - - - name: Archive network debugger database - uses: actions/upload-artifact@v4 - with: - name: network-debugger-test-bootstrap - path: /tmp/db - if: ${{ always() }} + git clone --depth 1 https://github.com/openmina/circuit-blobs.git + rm -rf circuit-blobs/.git circuit-blobs/berkeley_rc1 circuit-blobs/3.0.0mainnet + zip -0 -r circuit-blobs circuit-blobs + mv circuit-blobs.zip ledger/ + - run: | + cd ledger + wasm-pack test --release ${{ matrix.driver }} --headless -- -Z build-std=std,panic_abort + + # p2p-tests: + # runs-on: ubuntu-20.04 + # steps: + # - name: Git checkout + # uses: actions/checkout@v4 + + # - name: Setup build dependencies + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler + + # - name: Setup Rust + # run: | + # rustup default 1.80 + # rustup component add rustfmt + + # - name: Setup Rust Cache + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v0" + + # - name: Test p2p crate + # run: | + # cargo test -p p2p --tests + + + # build: + # runs-on: ubuntu-20.04 + # steps: + # - name: Git checkout + # uses: actions/checkout@v4 + + # - name: Setup build dependencies + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler + + # - name: Setup Rust + # run: | + # rustup default 1.80 + # rustup component add rustfmt + + # - name: Setup Rust Cache + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v0" + + # - name: Release build + # run: | + # cargo build --release --bin openmina + + # - name: Upload binaries + # uses: actions/upload-artifact@v4 + # with: + # name: bin + # path: target/release/openmina + + # build_wasm: + # runs-on: ubuntu-20.04 + # steps: + # - name: Git checkout + # uses: actions/checkout@v4 + + # - name: Setup build dependencies + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler + + # - name: Setup Rust + # run: | + # rustup default nightly + # rustup component add rustfmt rust-src + # rustup target add wasm32-unknown-unknown + # cargo install -f wasm-bindgen-cli --version 0.2.93 + + # - name: Setup Rust Cache + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v0" + + # - name: Release build + # run: | + # cd node/web + # cargo +nightly build --release --target wasm32-unknown-unknown + # wasm-bindgen --keep-debug --web --out-dir pkg ../../target/wasm32-unknown-unknown/release/openmina_node_web.wasm + + # build-tests: + # runs-on: ubuntu-20.04 + # steps: + # - name: Git checkout + # uses: actions/checkout@v4 + + # - name: Setup build dependencies + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler + + # - name: Setup Rust + # run: | + # rustup default 1.80 + # rustup component add rustfmt + + # - name: Setup Rust Cache + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v0" + + # - name: Build tests + # run: | + # mkdir -p target/release/tests + + # cargo build --release --tests --package=openmina-node-testing --package=cli + # cargo build --release --tests --package=openmina-node-testing --package=cli --message-format=json > cargo-build-test.json + # jq -r '. | select(.executable != null and (.target.kind | (contains(["test"])))) | [.target.name, .executable ] | @tsv' cargo-build-test.json > tests.tsv + # while read NAME FILE; do cp -a $FILE target/release/tests/$NAME; done < tests.tsv + + # - name: Upload tests + # uses: actions/upload-artifact@v4 + # with: + # name: tests + # path: target/release/tests + + # build-tests-webrtc: + # runs-on: ubuntu-20.04 + # steps: + # - name: Git checkout + # uses: actions/checkout@v4 + + # - name: Setup build dependencies + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler + + # - name: Setup Rust + # run: | + # rustup default 1.80 + # rustup component add rustfmt + + # - name: Setup Rust Cache + # uses: Swatinem/rust-cache@v2 + # with: + # prefix-key: "v0" + + # - name: Build tests + # run: | + # mkdir -p target/release/tests + + # cargo build --release --features=scenario-generators,p2p-webrtc --package=openmina-node-testing --tests + # cargo build --release --features=scenario-generators,p2p-webrtc --package=openmina-node-testing --tests --message-format=json > cargo-build-test.json + # jq -r '. | select(.executable != null and (.target.kind | (contains(["test"])))) | [.target.name, .executable ] | @tsv' cargo-build-test.json > tests.tsv + # while read NAME FILE; do cp -a $FILE target/release/tests/webrtc_$NAME; done < tests.tsv + + # - name: Upload tests + # uses: actions/upload-artifact@v4 + # with: + # name: tests-webrtc + # path: target/release/tests + + # p2p-scenario-tests: + # needs: [ build-tests, build-tests-webrtc ] + # runs-on: ubuntu-20.04 + # container: + # image: minaprotocol/mina-daemon:3.0.0-dc6bf78-focal-devnet + # options: --volume debugger_data:/tmp/db + # env: + # BPF_ALIAS: /coda/0.0.1/29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6-0.0.0.0 + # strategy: + # matrix: + # test: [p2p_basic_connections, p2p_basic_incoming, p2p_basic_outgoing, p2p_pubsub, p2p_kad, + # webrtc_p2p_basic_connections] + # fail-fast: false + + # services: + # network-debugger: + # image: openmina/mina-network-debugger:23385c61 + # options: --privileged --init --volume debugger_data:/tmp/db --volume /sys/kernel/debug:/sys/kernel/debug + # env: + # SERVER_PORT: 80 + # FIREWALL_INTERFACE: lo + # RUST_LOG: info + # DB_PATH: /tmp/db + # ports: + # - 80:80 + + # steps: + # - name: Download tests + # uses: actions/download-artifact@v4 + # with: + # pattern: tests* + # merge-multiple: true + + # - name: Setup permissions + # run: | + # chmod +x ./${{ matrix.test }} + + # # TODO: use curl + # - name: Wait for the debugger + # run: | + # sleep 5 + + # - name: Run the test + # run: | + # ./${{ matrix.test }} --test-threads=1 + + # - name: Archive network debugger database + # uses: actions/upload-artifact@v4 + # with: + # name: network-debugger-${{ matrix.test }} + # path: /tmp/db + # if: ${{ always() }} + + # k8s-peers: + # runs-on: ubuntu-20.04 + # # TODO: query cluster for actual addresses, or specify then on deployment + # env: + # PEERS: | + # /ip4/135.181.217.23/tcp/31881/p2p/12D3KooWS4TMSjrAS4Cj31PgjZ9KgeHh5goLP65M5GSriF28d7Jx + # /ip4/135.181.217.23/tcp/30386/p2p/12D3KooWK92cYz26JqBE9vM9s9Jd9pJKcNxLd7VVRupU7YG5NupU + # /ip4/135.181.217.23/tcp/32272/p2p/12D3KooWSU1DYZYVA7wAYLvLsH6yVS8oV1sMJUcC7VtUxCgtsWkJ + # /ip4/135.181.217.23/tcp/32218/p2p/12D3KooWM5m9QqHpDkPJi54GPP6rGFpzo7E274husrModPSLQ7tn + # /ip4/135.181.217.23/tcp/30798/p2p/12D3KooWCk2QSmQH2XbtpDXiSPUq6wb2LB2JaExHRXkJggmEfN4J + # /ip4/135.181.217.23/tcp/31631/p2p/12D3KooWQ1642Dzm57Kr8tmTwS9NRFaJPy4ysaQ2ne3ZYwQn5qCk + # /ip4/135.181.217.23/tcp/30196/p2p/12D3KooWHK67syE2LeTz5EnNqCe5ZFf9SoZRFN4AdHVZsL31WkMn + # /ip4/135.181.217.23/tcp/30790/p2p/12D3KooWDwxrG5u12FzXAFyK7vd8aHnEQf4dwoboBJ72FUS179xK + # /ip4/135.181.217.23/tcp/30070/p2p/12D3KooWEowA3VakSddUjZuBTK3HJhNM7sRqwWDbqtPtwymAMCcy + # outputs: + # peers: ${{ steps.peers.outputs.peers }} + # steps: + # - id: peers + # name: Fetch k8s OCaml peers + # run: | + # { + # echo 'peers<> "$GITHUB_OUTPUT" + + # scenario-tests: + # needs: + # - k8s-peers + # - build-tests + # # - build-tests-webrtc + # runs-on: ubuntu-20.04 + # container: + # image: minaprotocol/mina-daemon:3.0.0-dc6bf78-focal-devnet + # options: --volume debugger_data:/tmp/db + # env: + # # to allow local addrs discovery + # OPENMINA_DISCOVERY_FILTER_ADDR: false + # # to allow connection with replayer + # # TODO: remove when replayer supports identify + # KEEP_CONNECTION_WITH_UNKNOWN_STREAM: true + # OPENMINA_SCENARIO_SEEDS: ${{ needs.k8s-peers.outputs.peers }} + # BPF_ALIAS: /coda/0.0.1/29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6-0.0.0.0 + # strategy: + # matrix: + # test: + # - single_node + # - multi_node_initial_joining + # - multi_node_peer_discovery + # - multi_node_propagate_block + # - connection_discovery_ocaml_to_rust_via_seed + # - connection_discovery_ocaml_to_rust + # - connection_discovery_rust_as_seed + # - connection_discovery_rust_to_ocaml_via_seed + # - connection_discovery_rust_to_ocaml + # # - webrtc_single_node + # # - webrtc_multi_node + # fail-fast: false + + # services: + # network-debugger: + # image: openmina/mina-network-debugger:23385c61 + # options: --privileged --init --volume debugger_data:/tmp/db --volume /sys/kernel/debug:/sys/kernel/debug + # env: + # SERVER_PORT: 80 + # FIREWALL_INTERFACE: lo + # RUST_LOG: info + # DB_PATH: /tmp/db + # ports: + # - 80:80 + + # steps: + # - name: Download tests + # uses: actions/download-artifact@v4 + # with: + # pattern: tests* + # merge-multiple: true + + # - name: Setup permissions + # run: | + # chmod +x ./${{ matrix.test }} + + # # TODO: use curl + # - name: Wait for the debugger + # run: | + # sleep 5 + + # - name: Run the test + # run: | + # ./${{ matrix.test }} --test-threads=1 + + # - name: Archive network debugger database + # uses: actions/upload-artifact@v4 + # with: + # name: network-debugger-${{ matrix.test }} + # path: /tmp/db + # if: ${{ always() }} + + # record-replay-tests: + # needs: + # - k8s-peers + # - build-tests + # - build-tests-webrtc + # runs-on: ubuntu-20.04 + # container: + # image: minaprotocol/mina-daemon:3.0.0-dc6bf78-focal-devnet + # env: + # # to allow local addrs discovery + # OPENMINA_DISCOVERY_FILTER_ADDR: false + # OPENMINA_SCENARIO_SEEDS: ${{ needs.k8s-peers.outputs.peers }} + # strategy: + # matrix: + # test: [record_replay, webrtc_record_replay] + # fail-fast: false + + # steps: + # - name: Download tests + # uses: actions/download-artifact@v4 + # with: + # pattern: tests* + # merge-multiple: true + + # - name: Setup permissions + # run: | + # chmod +x ./${{ matrix.test }} + + # - name: Run the test + # run: | + # ./${{ matrix.test }} --test-threads=1 + + + # bootstrap-test: + # needs: [ k8s-peers, build, build-tests ] + # runs-on: ubuntu-20.04 + # env: + # PEERS_LIST: ${{ needs.k8s-peers.outputs.peers }} + # PEER_LIST_FILE: peer-list.txt + # WORK_DIR: data + # BPF_ALIAS: /coda/0.0.1/29936104443aaf264a7f0192ac64b1c7173198c1ed404c1bcff5e562e05eb7f6-0.0.0.0 + + # services: + # network-debugger: + # image: openmina/mina-network-debugger:23385c61 + # options: --privileged --init --volume /tmp/db:/tmp/db --volume /sys/kernel/debug:/sys/kernel/debug + # env: + # SERVER_PORT: 80 + # FIREWALL_INTERFACE: lo + # RUST_LOG: info + # DB_PATH: /tmp/db + # ports: + # - 80:80 + + # steps: + # - name: Download binary + # uses: actions/download-artifact@v4 + # with: + # name: bin + + # - name: Download test + # uses: actions/download-artifact@v4 + # with: + # pattern: tests* + # merge-multiple: true + + # - name: Fix permissions + # run: | + # chmod +x bootstrap openmina + + # # TODO: use curl + # - name: Wait for the debugger + # run: | + # sleep 5 + + # - name: Peer List File + # run: | + # for PEER in $PEERS_LIST; do echo $PEER; done > $PEER_LIST_FILE + # cat $PEER_LIST_FILE + + # - name: Bootstrap node + # env: + # OPENMINA_COMMAND: openmina + # NO_PEER_DISCOVERY: "true" + # OUT_PATH: ${{ env.WORK_DIR }}/logs/bootstrap_output + # RECORD: state-with-input-actions + # run: | + # mkdir -p $OUT_PATH + # PATH=$PATH:$(pwd) OPENMINA_COMMAND=openmina NO_PEER_DISCOVERY=true ./bootstrap --nocapture || { + # echo "::group::Stderr" + # cat $OUT_PATH.stderr + # echo "::endgroup::" + # exit 1 + # } + + # - name: Upload logs + # uses: actions/upload-artifact@v4 + # with: + # name: bootstrap-logs + # path: ${{ env.WORK_DIR }}/logs/* + # if: ${{ failure() }} + + # - name: Upload record + # uses: actions/upload-artifact@v4 + # with: + # name: bootstrap-record + # path: ${{ env.WORK_DIR }}/recorder/* + # if: ${{ failure() }} + + # - name: Archive network debugger database + # uses: actions/upload-artifact@v4 + # with: + # name: network-debugger-test-bootstrap + # path: /tmp/db + # if: ${{ always() }} diff --git a/Cargo.lock b/Cargo.lock index 642bf626c..6d666291d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aead" version = "0.4.3" @@ -50,9 +56,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher 0.4.4", @@ -80,7 +86,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead 0.5.2", - "aes 0.8.3", + "aes 0.8.4", "cipher 0.4.4", "ctr 0.9.2", "ghash 0.5.0", @@ -221,6 +227,15 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.6.0" @@ -607,7 +622,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.1", "object 0.32.1", "rustc-demangle", ] @@ -811,9 +826,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -841,12 +856,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -1164,9 +1180,9 @@ checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484" [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -1213,12 +1229,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -1518,6 +1531,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "derive_builder" version = "0.11.2" @@ -1635,9 +1659,9 @@ checksum = "d102f1a462fdcdddce88d6d46c06c074a2d2749b262230333726b06c52bb7585" [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", @@ -1912,13 +1936,13 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "libz-sys", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -2849,18 +2873,18 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -3421,9 +3445,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "pkg-config", @@ -3553,9 +3577,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -3757,6 +3781,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-bindgen-test", "web-sys", + "zip", "zstd", ] @@ -3775,6 +3800,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -6306,6 +6340,12 @@ dependencies = [ "dirs", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -6920,18 +6960,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -7650,20 +7690,19 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", - "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", @@ -7676,9 +7715,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -7688,9 +7727,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7698,9 +7737,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", @@ -7711,9 +7750,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-bindgen-test" @@ -7837,7 +7876,7 @@ name = "webrtc-dtls" version = "0.10.0" source = "git+https://github.com/openmina/webrtc.git?branch=openmina-v0.11.0#6f1a6dd2dc888d4b1ad5cec61be321f12fcb44a7" dependencies = [ - "aes 0.8.3", + "aes 0.8.4", "aes-gcm 0.10.3", "async-trait", "bincode", @@ -7939,7 +7978,7 @@ version = "0.13.0" source = "git+https://github.com/openmina/webrtc.git?branch=openmina-v0.11.0#6f1a6dd2dc888d4b1ad5cec61be321f12fcb44a7" dependencies = [ "aead 0.5.2", - "aes 0.8.3", + "aes 0.8.4", "aes-gcm 0.10.3", "byteorder", "bytes", @@ -8366,9 +8405,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] @@ -8384,6 +8423,21 @@ dependencies = [ "syn 2.0.58", ] +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "indexmap 2.0.2", + "memchr", + "thiserror", +] + [[package]] name = "zstd" version = "0.12.4" @@ -8405,9 +8459,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index 8dd992e76..e59cb3d36 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -72,6 +72,7 @@ serde_with = "3.6.1" anyhow = "1.0.75" thiserror = "1.0.60" fraction = { version = "=0.15.1", default-features = false, features = ["with-serde-support"] } +zip = { version = "2.2", default-features = false } [target.'cfg(target_family = "wasm")'.dependencies] getrandom = { version = "0.2", features = ["js"] } @@ -93,7 +94,8 @@ rand_seeder = "0.2" tuple-map = "0.4.0" [target.'cfg(target_family = "wasm")'.dev-dependencies] -wasm-bindgen-test = "0.3.0" +wasm-bindgen = "=0.2.92" +wasm-bindgen-test = "0.3" web-sys = { version = "0.3", features = ["Blob", "DedicatedWorkerGlobalScope", "MessageEvent", "Url", "Worker", "WorkerType", "WorkerOptions", "console", "Window", "Performance" ] } [features] diff --git a/ledger/src/account/account.rs b/ledger/src/account/account.rs index b5dbe4e1c..71c59eb31 100644 --- a/ledger/src/account/account.rs +++ b/ledger/src/account/account.rs @@ -1759,9 +1759,8 @@ mod tests { #[cfg(not(target_family = "wasm"))] const SIZE: usize = 280; - // FIXME: was 2496bytes before zkapp got boxed, what should be the size now? #[cfg(target_family = "wasm")] - const SIZE: usize = 280; + const SIZE: usize = 264; assert_eq!(std::mem::size_of::(), SIZE); } diff --git a/ledger/src/proofs/circuit_blobs.rs b/ledger/src/proofs/circuit_blobs.rs index 23ec9c21a..32aee249f 100644 --- a/ledger/src/proofs/circuit_blobs.rs +++ b/ledger/src/proofs/circuit_blobs.rs @@ -52,67 +52,96 @@ fn git_release_url(filename: &impl AsRef) -> String { format!("{RELEASES_PATH}/{filename_str}") } -#[cfg(not(target_family = "wasm"))] -pub fn fetch(filename: &impl AsRef) -> std::io::Result> { - use std::path::PathBuf; +// #[cfg(not(target_family = "wasm"))] +// pub async fn fetch(filename: &impl AsRef) -> std::io::Result> { +// use std::path::PathBuf; + +// fn try_base_dir>(base_dir: P, filename: &impl AsRef) -> Option { +// let mut path = base_dir.into(); +// path.push(filename); +// path.exists().then_some(path) +// } + +// fn to_io_err(err: impl std::fmt::Display) -> std::io::Error { +// std::io::Error::new( +// std::io::ErrorKind::Other, +// format!( +// "failed to find circuit-blobs locally and to fetch the from github! error: {err}" +// ), +// ) +// } + +// let home_base_dir = home_base_dir(); +// let found = None +// .or_else(|| { +// try_base_dir( +// std::env::var("OPENMINA_CIRCUIT_BLOBS_BASE_DIR").ok()?, +// filename, +// ) +// }) +// .or_else(|| try_base_dir(env!("CARGO_MANIFEST_DIR").to_string(), filename)) +// .or_else(|| try_base_dir(home_base_dir.clone()?, filename)) +// .or_else(|| try_base_dir("/usr/local/lib/openmina/circuit-blobs", filename)); + +// if let Some(path) = found { +// return std::fs::read(path); +// } + +// eprintln!( +// "circuit-blobs '{}' not found locally, so fetching it...", +// filename.as_ref().to_str().unwrap() +// ); +// let base_dir = home_base_dir.expect("$HOME env not set!"); + +// let bytes = reqwest::blocking::get(git_release_url(filename)) +// .map_err(to_io_err)? +// .bytes() +// .map_err(to_io_err)? +// .to_vec(); + +// // cache it to home dir. +// let cache_path = base_dir.join(filename); +// eprintln!("caching circuit-blobs to {}", cache_path.to_str().unwrap()); +// let _ = std::fs::create_dir_all(cache_path.parent().unwrap()); +// let _ = std::fs::write(cache_path, &bytes); + +// Ok(bytes) +// } + +// #[cfg(not(test))] +// #[cfg(target_family = "wasm")] +// pub async fn fetch(filename: &impl AsRef) -> std::io::Result> { +// let prefix = +// option_env!("CIRCUIT_BLOBS_HTTP_PREFIX").unwrap_or("/assets/webnode/circuit-blobs"); +// let url = format!("{prefix}/{}", filename.as_ref().to_str().unwrap()); +// http::get_bytes(&url).await +// // http::get_bytes(&git_release_url(filename)).await +// } + +// #[cfg(test)] +// #[cfg(target_family = "wasm")] +pub const CIRCUIT_BLOBS: &[u8] = include_bytes!("../../circuit-blobs.zip"); + +// #[cfg(test)] +// #[cfg(target_family = "wasm")] +pub async fn fetch(filename: &impl AsRef) -> std::io::Result> { + use std::io::Read; - fn try_base_dir>(base_dir: P, filename: &impl AsRef) -> Option { - let mut path = base_dir.into(); - path.push(filename); - path.exists().then_some(path) - } + const MAX_SIZE: usize = 32 * 1024 * 1024; + let mut file = std::io::Cursor::new(CIRCUIT_BLOBS); + let mut zip = zip::ZipArchive::new(&mut file).unwrap(); - fn to_io_err(err: impl std::fmt::Display) -> std::io::Error { - std::io::Error::new( - std::io::ErrorKind::Other, - format!( - "failed to find circuit-blobs locally and to fetch the from github! error: {err}" - ), - ) - } + let filename = filename.as_ref().to_str().unwrap(); + eprintln!("fetching {:?}", filename); - let home_base_dir = home_base_dir(); - let found = None - .or_else(|| { - try_base_dir( - std::env::var("OPENMINA_CIRCUIT_BLOBS_BASE_DIR").ok()?, - filename, - ) - }) - .or_else(|| try_base_dir(env!("CARGO_MANIFEST_DIR").to_string(), filename)) - .or_else(|| try_base_dir(home_base_dir.clone()?, filename)) - .or_else(|| try_base_dir("/usr/local/lib/openmina/circuit-blobs", filename)); - - if let Some(path) = found { - return std::fs::read(path); + for i in 0..zip.len() { + let file = zip.by_index(i).unwrap(); + if file.name().ends_with(filename) { + let mut buffer = Vec::with_capacity(MAX_SIZE); + file.take(MAX_SIZE as _).read_to_end(&mut buffer).unwrap(); + return Ok(buffer); + } } - eprintln!( - "circuit-blobs '{}' not found locally, so fetching it...", - filename.as_ref().to_str().unwrap() - ); - let base_dir = home_base_dir.expect("$HOME env not set!"); - - let bytes = reqwest::blocking::get(git_release_url(filename)) - .map_err(to_io_err)? - .bytes() - .map_err(to_io_err)? - .to_vec(); - - // cache it to home dir. - let cache_path = base_dir.join(filename); - eprintln!("caching circuit-blobs to {}", cache_path.to_str().unwrap()); - let _ = std::fs::create_dir_all(cache_path.parent().unwrap()); - let _ = std::fs::write(cache_path, &bytes); - - Ok(bytes) -} - -#[cfg(target_family = "wasm")] -pub async fn fetch(filename: &impl AsRef) -> std::io::Result> { - let prefix = - option_env!("CIRCUIT_BLOBS_HTTP_PREFIX").unwrap_or("/assets/webnode/circuit-blobs"); - let url = format!("{prefix}/{}", filename.as_ref().to_str().unwrap()); - http::get_bytes(&url).await - // http::get_bytes(&git_release_url(filename)).await + panic!("{} not found in zip", filename); } diff --git a/ledger/src/proofs/provers.rs b/ledger/src/proofs/provers.rs index 1e1da34c1..8f4b62627 100644 --- a/ledger/src/proofs/provers.rs +++ b/ledger/src/proofs/provers.rs @@ -9,8 +9,9 @@ use openmina_core::network::CircuitsConfig; use super::{ circuit_blobs, constants::{ - StepBlockProof, StepMergeProof, StepTransactionProof, StepZkappOptSignedOptSignedProof, - StepZkappOptSignedProof, StepZkappProvedProof, WrapBlockProof, WrapTransactionProof, + ProofConstants, StepBlockProof, StepMergeProof, StepTransactionProof, + StepZkappOptSignedOptSignedProof, StepZkappOptSignedProof, StepZkappProvedProof, + WrapBlockProof, WrapTransactionProof, }, field::FieldWitness, transaction::{make_prover_index, InternalVars, Prover, V}, @@ -22,7 +23,7 @@ use mina_p2p_messages::binprot::{ macros::{BinProtRead, BinProtWrite}, }; -pub fn devnet_circuit_directory() -> &'static str { +pub const fn devnet_circuit_directory() -> &'static str { openmina_core::network::devnet::CIRCUITS_CONFIG.directory_name } @@ -50,15 +51,6 @@ fn decode_gates_file( Ok(data.gates) } -#[cfg(not(target_family = "wasm"))] -fn read_gates_file( - filename: &impl AsRef, -) -> std::io::Result>> { - let bytes = circuit_blobs::fetch(filename)?; - decode_gates_file(bytes.as_slice()) -} - -#[cfg(target_family = "wasm")] async fn read_gates_file( filepath: &impl AsRef, ) -> std::io::Result>> { @@ -66,29 +58,6 @@ async fn read_gates_file( decode_gates_file(&mut resp.as_slice()) } -#[cfg(not(target_family = "wasm"))] -fn make_gates( - filename: &str, -) -> ( - HashMap, Option)>, - Vec>>, - Vec>, -) { - let circuits_config = openmina_core::NetworkConfig::global().circuits_config; - let base_dir = Path::new(&circuits_config.directory_name); - - let internal_vars_path = base_dir.join(format!("{}_internal_vars.bin", filename)); - let rows_rev_path = base_dir.join(format!("{}_rows_rev.bin", filename)); - let gates_path = base_dir.join(format!("{}_gates.json", filename)); - - let gates: Vec> = read_gates_file(&gates_path).unwrap(); - let (internal_vars_path, rows_rev_path) = - read_constraints_data::(&internal_vars_path, &rows_rev_path).unwrap(); - - (internal_vars_path, rows_rev_path, gates) -} - -#[cfg(target_family = "wasm")] async fn make_gates( filename: &str, ) -> ( @@ -112,32 +81,47 @@ async fn make_gates( (internal_vars_path, rows_rev_path, gates) } -macro_rules! get_or_make { - ($constant: ident, $type: ty, $filename: expr) => {{ - get_or_make!($constant, $type, None, $filename) - }}; - ($constant: ident, $type: ty, $verifier_index: expr, $filename: expr) => {{ - if let Some(prover) = $constant.get() { - return prover.clone(); +async fn get_or_make( + constant: &OnceCell>>, + verifier_index: Option>>, + filename: &str, +) -> Arc> { + if let Some(prover) = constant.get() { + return prover.clone(); + } + + let (internal_vars, rows_rev, gates) = { make_gates(filename).await }; + + let index = make_prover_index::(gates, verifier_index); + let prover = Prover { + internal_vars, + rows_rev, + index, + }; + + constant.get_or_init(|| prover.into()).clone() +} + +/// Run the future, this does not support awakes +pub fn block_on(future: F) -> F::Output +where + F: std::future::Future, +{ + use std::task::{Context, Poll, Wake, Waker}; + + struct NoAwake; + impl Wake for NoAwake { + fn wake(self: Arc) { + unreachable!() // We don't support that } + } - let (internal_vars, rows_rev, gates) = { - #[cfg(not(target_family = "wasm"))] - let res = make_gates($filename); - #[cfg(target_family = "wasm")] - let res = make_gates($filename).await; - res - }; - - let index = make_prover_index::<$type, _>(gates, $verifier_index); - let prover = Prover { - internal_vars, - rows_rev, - index, - }; - - $constant.get_or_init(|| prover.into()).clone() - }}; + let mut future = std::pin::pin!(future); + let waker = Waker::from(Arc::new(NoAwake)); + match future.as_mut().poll(&mut Context::from_waker(&waker)) { + Poll::Ready(result) => return result, + Poll::Pending => unreachable!(), // We don't support that + } } static TX_STEP_PROVER: OnceCell>> = OnceCell::new(); @@ -153,196 +137,70 @@ fn default_circuits_config() -> &'static CircuitsConfig { openmina_core::NetworkConfig::global().circuits_config } -#[cfg(not(target_family = "wasm"))] -mod prover_makers { - use super::*; - - fn get_or_make_tx_step_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - TX_STEP_PROVER, - StepTransactionProof, - config.step_transaction_gates - ) - } - fn get_or_make_tx_wrap_prover( - config: &CircuitsConfig, - verifier_index: Option, - ) -> Arc> { - get_or_make!( - TX_WRAP_PROVER, - WrapTransactionProof, - verifier_index.map(Into::into), - config.wrap_transaction_gates - ) - } - fn get_or_make_merge_step_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!(MERGE_STEP_PROVER, StepMergeProof, config.step_merge_gates) - } - fn get_or_make_block_step_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - BLOCK_STEP_PROVER, - StepBlockProof, - config.step_blockchain_gates - ) - } - fn get_or_make_block_wrap_prover( - config: &CircuitsConfig, - verifier_index: Option, - ) -> Arc> { - get_or_make!( - BLOCK_WRAP_PROVER, - WrapBlockProof, - verifier_index.map(Into::into), - config.wrap_blockchain_gates - ) - } - fn get_or_make_zkapp_step_opt_signed_opt_signed_prover( - config: &CircuitsConfig, - ) -> Arc> { - get_or_make!( - ZKAPP_STEP_OPT_SIGNED_OPT_SIGNED_PROVER, - StepZkappOptSignedOptSignedProof, - config.step_transaction_opt_signed_opt_signed_gates - ) - } - fn get_or_make_zkapp_step_opt_signed_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - ZKAPP_STEP_OPT_SIGNED_PROVER, - StepZkappOptSignedProof, - config.step_transaction_opt_signed_gates - ) - } - fn get_or_make_zkapp_step_proof_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - ZKAPP_STEP_PROOF_PROVER, - StepZkappProvedProof, - config.step_transaction_proved_gates - ) - } - - impl BlockProver { - pub fn make( - block_verifier_index: Option, - tx_verifier_index: Option, - ) -> Self { - let config = default_circuits_config(); - let block_step_prover = get_or_make_block_step_prover(config); - let block_wrap_prover = get_or_make_block_wrap_prover(config, block_verifier_index); - let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index); - - Self { - block_step_prover, - block_wrap_prover, - tx_wrap_prover, - } - } - } - - impl TransactionProver { - pub fn make(tx_verifier_index: Option) -> Self { - let config = default_circuits_config(); - let tx_step_prover = get_or_make_tx_step_prover(config); - let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index); - let merge_step_prover = get_or_make_merge_step_prover(config); - - Self { - tx_step_prover, - tx_wrap_prover, - merge_step_prover, - } - } - } - - impl ZkappProver { - pub fn make(tx_verifier_index: Option) -> Self { - let config = default_circuits_config(); - let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index); - let merge_step_prover = get_or_make_merge_step_prover(config); - let step_opt_signed_opt_signed_prover = - get_or_make_zkapp_step_opt_signed_opt_signed_prover(config); - let step_opt_signed_prover = get_or_make_zkapp_step_opt_signed_prover(config); - let step_proof_prover = get_or_make_zkapp_step_proof_prover(config); - - Self { - tx_wrap_prover, - merge_step_prover, - step_opt_signed_opt_signed_prover, - step_opt_signed_prover, - step_proof_prover, - } - } - } +async fn get_or_make_tx_step_prover(config: &CircuitsConfig) -> Arc> { + get_or_make::(&TX_STEP_PROVER, None, config.step_transaction_gates) + .await +} +async fn get_or_make_tx_wrap_prover( + config: &CircuitsConfig, + verifier_index: Option, +) -> Arc> { + get_or_make::( + &TX_WRAP_PROVER, + verifier_index.map(Into::into), + config.wrap_transaction_gates, + ) + .await +} +async fn get_or_make_merge_step_prover(config: &CircuitsConfig) -> Arc> { + get_or_make::(&MERGE_STEP_PROVER, None, config.step_merge_gates).await +} +async fn get_or_make_block_step_prover(config: &CircuitsConfig) -> Arc> { + get_or_make::(&BLOCK_STEP_PROVER, None, config.step_blockchain_gates).await +} +async fn get_or_make_block_wrap_prover( + config: &CircuitsConfig, + verifier_index: Option, +) -> Arc> { + get_or_make::( + &BLOCK_WRAP_PROVER, + verifier_index.map(Into::into), + config.wrap_blockchain_gates, + ) + .await +} +async fn get_or_make_zkapp_step_opt_signed_opt_signed_prover( + config: &CircuitsConfig, +) -> Arc> { + get_or_make::( + &ZKAPP_STEP_OPT_SIGNED_OPT_SIGNED_PROVER, + None, + config.step_transaction_opt_signed_opt_signed_gates, + ) + .await +} +async fn get_or_make_zkapp_step_opt_signed_prover(config: &CircuitsConfig) -> Arc> { + get_or_make::( + &ZKAPP_STEP_OPT_SIGNED_PROVER, + None, + config.step_transaction_opt_signed_gates, + ) + .await +} +async fn get_or_make_zkapp_step_proof_prover(config: &CircuitsConfig) -> Arc> { + get_or_make::( + &ZKAPP_STEP_PROOF_PROVER, + None, + config.step_transaction_proved_gates, + ) + .await } -#[cfg(target_family = "wasm")] mod prover_makers { use super::*; - async fn get_or_make_tx_step_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - TX_STEP_PROVER, - StepTransactionProof, - config.step_transaction_gates - ) - } - async fn get_or_make_tx_wrap_prover( - config: &CircuitsConfig, - verifier_index: Option, - ) -> Arc> { - get_or_make!( - TX_WRAP_PROVER, - WrapTransactionProof, - verifier_index.map(Into::into), - config.wrap_transaction_gates - ) - } - async fn get_or_make_merge_step_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!(MERGE_STEP_PROVER, StepMergeProof, config.step_merge_gates) - } - async fn get_or_make_block_step_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - BLOCK_STEP_PROVER, - StepBlockProof, - config.step_blockchain_gates - ) - } - async fn get_or_make_block_wrap_prover( - config: &CircuitsConfig, - verifier_index: Option, - ) -> Arc> { - get_or_make!( - BLOCK_WRAP_PROVER, - WrapBlockProof, - verifier_index.map(Into::into), - config.wrap_blockchain_gates - ) - } - async fn get_or_make_zkapp_step_opt_signed_opt_signed_prover( - config: &CircuitsConfig, - ) -> Arc> { - get_or_make!( - ZKAPP_STEP_OPT_SIGNED_OPT_SIGNED_PROVER, - StepZkappOptSignedOptSignedProof, - config.step_transaction_opt_signed_opt_signed_gates - ) - } - async fn get_or_make_zkapp_step_opt_signed_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - ZKAPP_STEP_OPT_SIGNED_PROVER, - StepZkappOptSignedProof, - config.step_transaction_opt_signed_gates - ) - } - async fn get_or_make_zkapp_step_proof_prover(config: &CircuitsConfig) -> Arc> { - get_or_make!( - ZKAPP_STEP_PROOF_PROVER, - StepZkappProvedProof, - config.step_transaction_proved_gates - ) - } - impl BlockProver { - pub async fn make( + pub async fn make_async( block_verifier_index: Option, tx_verifier_index: Option, ) -> Self { @@ -358,10 +216,18 @@ mod prover_makers { tx_wrap_prover, } } + + #[cfg(not(target_family = "wasm"))] + pub fn make( + block_verifier_index: Option, + tx_verifier_index: Option, + ) -> Self { + block_on(Self::make_async(block_verifier_index, tx_verifier_index)) + } } impl TransactionProver { - pub async fn make(tx_verifier_index: Option) -> Self { + pub async fn make_async(tx_verifier_index: Option) -> Self { let config = default_circuits_config(); let tx_step_prover = get_or_make_tx_step_prover(config).await; let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index).await; @@ -373,10 +239,15 @@ mod prover_makers { merge_step_prover, } } + + #[cfg(not(target_family = "wasm"))] + pub fn make(tx_verifier_index: Option) -> Self { + block_on(Self::make_async(tx_verifier_index)) + } } impl ZkappProver { - pub async fn make(tx_verifier_index: Option) -> Self { + pub async fn make_async(tx_verifier_index: Option) -> Self { let config = default_circuits_config(); let tx_wrap_prover = get_or_make_tx_wrap_prover(config, tx_verifier_index).await; let merge_step_prover = get_or_make_merge_step_prover(config).await; @@ -393,6 +264,11 @@ mod prover_makers { step_proof_prover, } } + + #[cfg(not(target_family = "wasm"))] + pub fn make(tx_verifier_index: Option) -> Self { + block_on(Self::make_async(tx_verifier_index)) + } } } @@ -471,21 +347,6 @@ fn decode_constraints_data( Some((internal_vars, rows_rev)) } -#[cfg(not(target_family = "wasm"))] -fn read_constraints_data( - internal_vars_path: &Path, - rows_rev_path: &Path, -) -> Option<(InternalVars, Vec>>)> { - // ((Fp.t * V.t) list * Fp.t option) - let internal_vars = circuit_blobs::fetch(&internal_vars_path).ok()?; - - // V.t option array list - let rows_rev = circuit_blobs::fetch(&rows_rev_path).ok()?; - - decode_constraints_data(internal_vars.as_slice(), rows_rev.as_slice()) -} - -#[cfg(target_family = "wasm")] async fn read_constraints_data( internal_vars_path: &Path, rows_rev_path: &Path, diff --git a/ledger/src/proofs/transaction.rs b/ledger/src/proofs/transaction.rs index 2ba60c89b..6547773d8 100644 --- a/ledger/src/proofs/transaction.rs +++ b/ledger/src/proofs/transaction.rs @@ -4182,6 +4182,10 @@ mod tests_with_wasm { #[cfg(test)] pub(super) mod tests { + #[cfg(target_family = "wasm")] + use wasm_bindgen_test::wasm_bindgen_test as test; + + use core::str; use std::path::Path; use mina_p2p_messages::binprot::{ @@ -4216,6 +4220,9 @@ pub(super) mod tests { } pub fn panic_in_ci() { + #[cfg(target_family = "wasm")] + panic!("missing circuit files !"); + fn is_ci() -> bool { std::env::var("CI").is_ok() } @@ -4255,6 +4262,36 @@ pub(super) mod tests { .collect() } + // #[cfg(not(target_family = "wasm"))] + // fn fetch_file(filename: &str) -> std::io::Result> { + // let path = Path::new(env!("CARGO_MANIFEST_DIR")) + // .join(devnet_circuit_directory()) + // .join("tests") + // .join(filename); + // std::fs::read(path) + // } + // #[cfg(target_family = "wasm")] + pub fn fetch_file(filename: &str) -> std::io::Result> { + use crate::proofs::circuit_blobs::CIRCUIT_BLOBS; + use std::io::Read; + + let mut file = std::io::Cursor::new(CIRCUIT_BLOBS); + let mut zip = zip::ZipArchive::new(&mut file).unwrap(); + + let filename = format!("{}/tests/{}", devnet_circuit_directory(), filename); + const MAX_SIZE: usize = 128 * 1024; + + for i in 0..zip.len() { + let file = zip.by_index(i).unwrap(); + if file.name().ends_with(&filename) { + let mut buffer = Vec::with_capacity(MAX_SIZE); + file.take(MAX_SIZE as _).read_to_end(&mut buffer).unwrap(); + return Ok(buffer); + } + } + panic!(); + } + #[allow(unused)] #[test] #[ignore] @@ -4274,6 +4311,10 @@ pub(super) mod tests { .join(devnet_circuit_directory()) .join("tests"); + if !path.exists() { + return; + } + let entries = std::fs::read_dir(path) .unwrap() .map(|res| res.map(|e| e.path())) @@ -4391,11 +4432,19 @@ pub(super) mod tests { (statement, [p1, p2], message) } + fn prover_wrapper(future: F) -> F::Output { + crate::proofs::provers::block_on(future) + } + + #[cfg(not(target_family = "wasm"))] #[allow(unused)] #[test] fn test_make_verifier_index() { - BlockProver::make(None, None); - TransactionProver::make(None); + prover_wrapper(BlockProver::make_async(None, None)); + prover_wrapper(TransactionProver::make_async(None)); + + // get_block_prover(); + // get_tx_prover(); // use crate::proofs::caching::verifier_index_to_bytes; // use crate::proofs::verifier_index::get_verifier_index; @@ -4457,16 +4506,17 @@ pub(super) mod tests { // assert_eq!(format!("{:?}", v.lookup_index), format!("{:?}", new_v.lookup_index)); } + #[cfg(not(target_family = "wasm"))] // same test in `test_proofs` #[test] fn test_regular_tx() { - let Ok(data) = - // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("request_signed.bin")) - // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("rampup4").join("request_payment_0_rampup4.bin")) - std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(devnet_circuit_directory()).join("tests").join("command-0-1.bin")) - // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("rampup4").join("request_payment_1_rampup4.bin")) - // std::fs::read("/tmp/fee_transfer_1_rampup4.bin") - // std::fs::read("/tmp/coinbase_1_rampup4.bin") - // std::fs::read("/tmp/stake_0_rampup4.bin") + let Ok(data) = fetch_file("command-0-1.bin") + // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("request_signed.bin")) + // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("rampup4").join("request_payment_0_rampup4.bin")) + // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(devnet_circuit_directory()).join("tests").join("command-0-1.bin")) + // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("rampup4").join("request_payment_1_rampup4.bin")) + // std::fs::read("/tmp/fee_transfer_1_rampup4.bin") + // std::fs::read("/tmp/coinbase_1_rampup4.bin") + // std::fs::read("/tmp/stake_0_rampup4.bin") else { eprintln!("request not found"); panic_in_ci(); @@ -4478,7 +4528,7 @@ pub(super) mod tests { tx_step_prover, tx_wrap_prover, merge_step_prover: _, - } = TransactionProver::make(None); + } = prover_wrapper(TransactionProver::make_async(None)); let mut witnesses: Witness = Witness::new::(); // witnesses.ocaml_aux = read_witnesses("tx_fps.txt").unwrap(); @@ -4583,12 +4633,15 @@ pub(super) mod tests { } } + #[cfg(not(target_family = "wasm"))] // same test in `test_proofs` #[test] fn test_merge_proof() { let Ok(data) = // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("request_signed.bin")) // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("rampup4").join("merge_0_rampup4.bin")) - std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(devnet_circuit_directory()).join("tests").join("merge-100-0.bin")) + fetch_file("merge-100-0.bin") + + // std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join(devnet_circuit_directory()).join("tests").join("merge-100-0.bin")) // std::fs::read("/tmp/minaa/mina-works-dump/merge-100-0.bin") // std::fs::read("/tmp/fee_transfer_1_rampup4.bin") // std::fs::read("/tmp/coinbase_1_rampup4.bin") @@ -4604,7 +4657,7 @@ pub(super) mod tests { tx_step_prover: _, tx_wrap_prover, merge_step_prover, - } = TransactionProver::make(None); + } = prover_wrapper(TransactionProver::make_async(None)); let mut witnesses: Witness = Witness::new::(); // witnesses.ocaml_aux = read_witnesses("fps_merge.txt").unwrap(); @@ -4632,14 +4685,10 @@ pub(super) mod tests { let _sum = dbg!(sha256_sum(&proof_json)); } + #[cfg(not(target_family = "wasm"))] // same test in `test_proofs` #[test] fn test_proof_zkapp_sig() { - let Ok(data) = std::fs::read( - Path::new(env!("CARGO_MANIFEST_DIR")) - .join(devnet_circuit_directory()) - .join("tests") - .join("command-1-0.bin"), - ) else { + let Ok(data) = fetch_file("command-1-0.bin") else { eprintln!("request not found"); panic_in_ci(); return; @@ -4653,7 +4702,7 @@ pub(super) mod tests { step_opt_signed_opt_signed_prover, step_opt_signed_prover, step_proof_prover, - } = ZkappProver::make(None); + } = prover_wrapper(ZkappProver::make_async(None)); dbg!(step_opt_signed_opt_signed_prover.rows_rev.len()); // dbg!(step_opt_signed_opt_signed_prover.rows_rev.iter().map(|v| v.len()).collect::>()); @@ -4677,14 +4726,10 @@ pub(super) mod tests { let _sum = dbg!(sha256_sum(&proof_json)); } + #[cfg(not(target_family = "wasm"))] // same test in `test_proofs` #[test] fn test_proof_zkapp_proof() { - let Ok(data) = std::fs::read( - Path::new(env!("CARGO_MANIFEST_DIR")) - .join(devnet_circuit_directory()) - .join("tests") - .join("zkapp-command-with-proof-128-1.bin"), - ) else { + let Ok(data) = fetch_file("zkapp-command-with-proof-128-1.bin") else { eprintln!("request not found"); panic_in_ci(); return; @@ -4698,7 +4743,7 @@ pub(super) mod tests { step_opt_signed_opt_signed_prover, step_opt_signed_prover, step_proof_prover, - } = ZkappProver::make(None); + } = prover_wrapper(ZkappProver::make_async(None)); let LedgerProof { proof, .. } = generate_zkapp_proof(ZkappParams { statement: &statement, @@ -4720,14 +4765,10 @@ pub(super) mod tests { let _sum = dbg!(sha256_sum(&proof_json)); } + // #[cfg(not(target_family = "wasm"))] // same test in `test_proofs` #[test] fn test_block_proof() { - let Ok(data) = std::fs::read( - Path::new(env!("CARGO_MANIFEST_DIR")) - .join(devnet_circuit_directory()) - .join("tests") - .join("block_input-2483246-0.bin"), - ) else { + let Ok(data) = fetch_file("block_input-2483246-0.bin") else { eprintln!("request not found"); panic_in_ci(); return; @@ -4740,7 +4781,7 @@ pub(super) mod tests { block_step_prover, block_wrap_prover, tx_wrap_prover, - } = BlockProver::make(None, None); + } = prover_wrapper(BlockProver::make_async(None, None)); let mut witnesses: Witness = Witness::new::(); // witnesses.ocaml_aux = read_witnesses("block_fps.txt").unwrap(); @@ -4766,31 +4807,23 @@ pub(super) mod tests { let _sum = dbg!(sha256_sum(&proof_json)); } + #[cfg(not(target_family = "wasm"))] // same test in `test_proofs` #[test] fn test_proofs() { - let base_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .join(devnet_circuit_directory()) - .join("tests"); - - if !base_dir.exists() { - eprintln!("{:?} not found", base_dir); - panic_in_ci(); - return; - } - let BlockProver { block_step_prover, block_wrap_prover, tx_wrap_prover: _, - } = BlockProver::make(None, None); - let TransactionProver { tx_step_prover, .. } = TransactionProver::make(None); + } = prover_wrapper(BlockProver::make_async(None, None)); + let TransactionProver { tx_step_prover, .. } = + prover_wrapper(TransactionProver::make_async(None)); let ZkappProver { tx_wrap_prover, merge_step_prover, step_opt_signed_opt_signed_prover: zkapp_step_opt_signed_opt_signed_prover, step_opt_signed_prover: zkapp_step_opt_signed_prover, step_proof_prover: zkapp_step_proof_prover, - } = ZkappProver::make(None); + } = prover_wrapper(ZkappProver::make_async(None)); // TODO: Compare checksum with OCaml #[rustfmt::skip] @@ -4804,7 +4837,7 @@ pub(super) mod tests { ]; for (file, opt_signed_path, proved_path, expected_sum) in zkapp_cases { - let data = std::fs::read(base_dir.join(file)).unwrap(); + let data = fetch_file(file).unwrap(); let (statement, tx_witness, message) = extract_request(&data); let LedgerProof { proof, .. } = generate_zkapp_proof(ZkappParams { @@ -4833,7 +4866,7 @@ pub(super) mod tests { ("block_input-2483246-0.bin", None), // ("block_prove_inputs_7.bin", None), ] { - let data = std::fs::read(base_dir.join(filename)).unwrap(); + let data = fetch_file(filename).unwrap(); let blockchain_input: v2::ProverExtendBlockchainInputStableV2 = read_binprot(&mut data.as_slice()); @@ -4871,7 +4904,7 @@ pub(super) mod tests { // Merge proof { - let data = std::fs::read(base_dir.join("merge-100-0.bin")).unwrap(); + let data = fetch_file("merge-100-0.bin").unwrap(); let (statement, proofs, message) = extract_merge(&data); @@ -4935,7 +4968,7 @@ pub(super) mod tests { ]; for (file, expected_sum) in requests { - let data = std::fs::read(base_dir.join(file)).unwrap(); + let data = fetch_file(file).unwrap(); let (statement, tx_witness, message) = extract_request(&data); let mut witnesses: Witness = Witness::new::(); diff --git a/ledger/src/proofs/verification.rs b/ledger/src/proofs/verification.rs index cf23f6721..f218e4190 100644 --- a/ledger/src/proofs/verification.rs +++ b/ledger/src/proofs/verification.rs @@ -969,18 +969,16 @@ fn generate_new_filename(name: &str, extension: &str, data: &[u8]) -> std::io::R #[cfg(test)] mod tests { - use std::path::Path; + #[cfg(target_family = "wasm")] + use wasm_bindgen_test::wasm_bindgen_test as test; use mina_hasher::Fp; use mina_p2p_messages::{binprot::BinProtRead, v2}; - use crate::proofs::{provers::devnet_circuit_directory, transaction::tests::panic_in_ci}; + use crate::proofs::transaction::tests::{fetch_file, panic_in_ci}; use super::*; - #[cfg(target_family = "wasm")] - use wasm_bindgen_test::wasm_bindgen_test as test; - #[test] fn test_verify_zkapp() { use mina_p2p_messages::binprot; @@ -993,10 +991,6 @@ mod tests { proof: v2::PicklesProofProofsVerified2ReprStableV2, } - let base_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .join(devnet_circuit_directory()) - .join("tests"); - let cases = [ "verify_zapp_4af39d1e141859c964fe32b4e80537d3bd8c32d75e2754c0b869738006d25251_0.binprot", "verify_zapp_dc518dc7e0859ea6ffa0cd42637cdcc9c79ab369dfb7ff44c8a89b1219f98728_0.binprot", @@ -1005,7 +999,7 @@ mod tests { ]; for filename in cases { - let Ok(file) = std::fs::read(base_dir.join(filename)) else { + let Ok(file) = fetch_file(filename) else { panic_in_ci(); return; }; diff --git a/ledger/src/proofs/verifiers.rs b/ledger/src/proofs/verifiers.rs index 3ede6d9be..b75736bac 100644 --- a/ledger/src/proofs/verifiers.rs +++ b/ledger/src/proofs/verifiers.rs @@ -68,44 +68,28 @@ fn cache_path(kind: Kind) -> Option { super::circuit_blobs::home_base_dir().map(|p| p.join(cache_filename(kind))) } -macro_rules! read_cache { - ($kind: expr, $digest: expr) => {{ - #[cfg(not(target_family = "wasm"))] - let data = super::circuit_blobs::fetch(&cache_filename($kind)) - .context("fetching verifier index failed")?; - #[cfg(target_family = "wasm")] - let data = super::circuit_blobs::fetch(&cache_filename($kind)) - .await - .context("fetching verifier index failed")?; - let mut slice = data.as_slice(); - let mut d = [0; 32]; - // source digest - slice.read_exact(&mut d).context("reading source digest")?; - if d != $digest { - anyhow::bail!("source digest verification failed"); - } - - // index digest - slice.read_exact(&mut d).context("reading index digest")?; - - let mut hasher = Sha256::new(); - hasher.update(slice); - let digest = hasher.finalize(); - if d != digest.as_slice() { - anyhow::bail!("verifier index digest verification failed"); - } - Ok(super::caching::verifier_index_from_bytes(slice)?) - }}; -} +async fn read_cache(kind: Kind, digest: &[u8]) -> anyhow::Result> { + let data = super::circuit_blobs::fetch(&cache_filename(kind)) + .await + .context("fetching verifier index failed")?; + let mut slice = data.as_slice(); + let mut d = [0; 32]; + // source digest + slice.read_exact(&mut d).context("reading source digest")?; + if d != digest { + anyhow::bail!("source digest verification failed"); + } -#[cfg(not(target_family = "wasm"))] -fn read_cache(kind: Kind, digest: &[u8]) -> anyhow::Result> { - read_cache!(kind, digest) -} + // index digest + slice.read_exact(&mut d).context("reading index digest")?; -#[cfg(target_family = "wasm")] -async fn read_cache(kind: Kind, digest: &[u8]) -> anyhow::Result> { - read_cache!(kind, digest) + let mut hasher = Sha256::new(); + hasher.update(slice); + let digest = hasher.finalize(); + if d != digest.as_slice() { + anyhow::bail!("verifier index digest verification failed"); + } + Ok(super::caching::verifier_index_from_bytes(slice)?) } #[cfg(not(target_family = "wasm"))] @@ -130,46 +114,31 @@ fn write_cache(kind: Kind, index: &VerifierIndex, digest: &[u8]) -> anyhow:: Ok(()) } -macro_rules! make_with_ext_cache { - ($kind: expr, $data: expr) => {{ - let verifier_index: VerifierIndex = serde_json::from_str($data).unwrap(); - let mut hasher = Sha256::new(); - hasher.update($data); - let src_index_digest = hasher.finalize(); - - #[cfg(not(target_family = "wasm"))] - let cache = read_cache($kind, &src_index_digest); - #[cfg(target_family = "wasm")] - let cache = read_cache($kind, &src_index_digest).await; - - match cache { - Ok(verifier_index) => { - info!(system_time(); "Verifier index is loaded"); - verifier_index - } - Err(err) => { - warn!(system_time(); "Cannot load verifier index: {err}"); - let index = make_verifier_index(verifier_index); - #[cfg(not(target_family = "wasm"))] - if let Err(err) = write_cache($kind, &index, &src_index_digest) { - warn!(system_time(); "Cannot store verifier index to cache file: {err}"); - } else { - info!(system_time(); "Stored verifier index to cache file"); - } - index - } - } - }} -} +async fn make_with_ext_cache(kind: Kind, data: &str) -> VerifierIndex { + let verifier_index: VerifierIndex = serde_json::from_str(data).unwrap(); + let mut hasher = Sha256::new(); + hasher.update(data); + let src_index_digest = hasher.finalize(); -#[cfg(not(target_family = "wasm"))] -fn make_with_ext_cache(kind: Kind, data: &str) -> VerifierIndex { - make_with_ext_cache!(kind, data) -} + let cache = read_cache(kind, &src_index_digest).await; -#[cfg(target_family = "wasm")] -async fn make_with_ext_cache(kind: Kind, data: &str) -> VerifierIndex { - make_with_ext_cache!(kind, data) + match cache { + Ok(verifier_index) => { + info!(system_time(); "Verifier index is loaded"); + verifier_index + } + Err(err) => { + warn!(system_time(); "Cannot load verifier index: {err}"); + let index = make_verifier_index(verifier_index); + #[cfg(not(target_family = "wasm"))] + if let Err(err) = write_cache(kind, &index, &src_index_digest) { + warn!(system_time(); "Cannot store verifier index to cache file: {err}"); + } else { + info!(system_time(); "Stored verifier index to cache file"); + } + index + } + } } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -215,23 +184,8 @@ impl TransactionVerifier { } } -#[cfg(not(target_family = "wasm"))] -impl BlockVerifier { - pub fn make() -> Self { - BLOCK_VERIFIER - .get_or_init(|| { - Self(Arc::new(make_with_ext_cache( - Self::kind(), - Self::src_json(), - ))) - }) - .clone() - } -} - -#[cfg(target_family = "wasm")] impl BlockVerifier { - pub async fn make() -> Self { + pub async fn make_async() -> Self { if let Some(v) = BLOCK_VERIFIER.get() { v.clone() } else { @@ -241,25 +195,15 @@ impl BlockVerifier { BLOCK_VERIFIER.get_or_init(move || verifier).clone() } } -} -#[cfg(not(target_family = "wasm"))] -impl TransactionVerifier { + #[cfg(not(target_family = "wasm"))] pub fn make() -> Self { - TX_VERIFIER - .get_or_init(|| { - Self(Arc::new(make_with_ext_cache( - Self::kind(), - Self::src_json(), - ))) - }) - .clone() + super::provers::block_on(Self::make_async()) } } -#[cfg(target_family = "wasm")] impl TransactionVerifier { - pub async fn make() -> Self { + pub async fn make_async() -> Self { if let Some(v) = TX_VERIFIER.get() { v.clone() } else { @@ -269,6 +213,11 @@ impl TransactionVerifier { TX_VERIFIER.get_or_init(move || verifier).clone() } } + + #[cfg(not(target_family = "wasm"))] + pub fn make() -> Self { + super::provers::block_on(Self::make_async()) + } } impl std::ops::Deref for BlockVerifier { diff --git a/ledger/src/scan_state/scan_state.rs b/ledger/src/scan_state/scan_state.rs index e41c04174..2762482b6 100644 --- a/ledger/src/scan_state/scan_state.rs +++ b/ledger/src/scan_state/scan_state.rs @@ -2364,7 +2364,7 @@ impl ScanState { } } -pub fn group_list<'a, F, T, R>(slice: &'a [T], fun: F) -> impl Iterator> + '_ +pub fn group_list<'a, F, T, R>(slice: &'a [T], fun: F) -> impl Iterator> + 'a where F: Fn(&'a T) -> R + 'a, { diff --git a/ledger/src/staged_ledger/staged_ledger.rs b/ledger/src/staged_ledger/staged_ledger.rs index 9c18088ef..085beb51c 100644 --- a/ledger/src/staged_ledger/staged_ledger.rs +++ b/ledger/src/staged_ledger/staged_ledger.rs @@ -5949,6 +5949,10 @@ mod tests { assert_eq!(reference, serde_json::to_string(&hash).unwrap()); } + fn verifier_wrapper(future: F) -> F::Output { + crate::proofs::provers::block_on(future) + } + #[test] fn apply_berkeleynet() { #[allow(unused)] @@ -6013,7 +6017,8 @@ mod tests { pending_coinbase_collection: pending_coinbase, }; - let block_verifier = crate::proofs::verifiers::BlockVerifier::make(); + let block_verifier = + verifier_wrapper(crate::proofs::verifiers::BlockVerifier::make_async()); println!("initialized in {:?}", now.elapsed()); diff --git a/ledger/src/zkapps/zkapp_logic.rs b/ledger/src/zkapps/zkapp_logic.rs index bde54a57d..74c7c6fe0 100644 --- a/ledger/src/zkapps/zkapp_logic.rs +++ b/ledger/src/zkapps/zkapp_logic.rs @@ -41,11 +41,14 @@ pub enum ZkAppCommandElt { ZkAppCommandCommitment(crate::ReceiptChainHash), } -fn assert_(_b: Z::Bool) -> Result<(), String> { +fn assert_(b: Z::Bool, s: &str) -> Result<(), String> { // Used only for circuit generation (add constraints) // https://github.com/MinaProtocol/mina/blob/e44ddfe1ca54b3855e1ed336d89f6230d35aeb8c/src/lib/transaction_logic/zkapp_command_logic.ml#L929 - // TODO: In non-witness generation, we raise an exception + if let Boolean::False = b.as_boolean() { + return Err(s.to_string()); + } + Ok(()) } @@ -345,8 +348,8 @@ where let is_empty_call_forest = local_state.stack_frame.calls().is_empty(w); match is_start { IsStart::Compute(_) => (), - IsStart::Yes(_) => assert_::(is_empty_call_forest)?, - IsStart::No => assert_::(is_empty_call_forest.neg())?, + IsStart::Yes(_) => assert_::(is_empty_call_forest, "is_empty_call_forest")?, + IsStart::No => assert_::(is_empty_call_forest.neg(), "is_empty_call_forest.neg()")?, }; match is_start { IsStart::Yes(_) => Z::Bool::true_(), @@ -566,16 +569,14 @@ where w, ) }; - assert_::(Z::Bool::equal( - proof_verifies, - account_update.is_proved(), - w, - ))?; - assert_::(Z::Bool::equal( - signature_verifies, - account_update.is_signed(), - w, - ))?; + assert_::( + Z::Bool::equal(proof_verifies, account_update.is_proved(), w), + "not proved", + )?; + assert_::( + Z::Bool::equal(signature_verifies, account_update.is_signed(), w), + "not signed", + )?; Z::LocalState::add_check( local_state, @@ -650,11 +651,14 @@ where SetOrKeep::Keep => a.get().timing.clone(), } }); - assert_::(Z::GlobalSlotSpan::greater_than( - &timing.to_record().vesting_period, - &SlotSpan::zero(), - w, - ))?; + assert_::( + Z::GlobalSlotSpan::greater_than( + &timing.to_record().vesting_period, + &SlotSpan::zero(), + w, + ), + "vesting_period zero", + )?; a.get_mut().timing = timing; ((), ()) }; @@ -1184,7 +1188,7 @@ where Z::SignedAmount::is_non_neg(&local_delta), w, ); - assert_::(Z::Bool::or(is_start2.neg(), first, w))?; + assert_::(Z::Bool::or(is_start2.neg(), first, w), "is_start2 or first")?; let (new_local_fee_excess, overflow) = Z::SignedAmount::add_flagged(&local_state.excess, &local_delta, w); // We decompose this way because of OCaml evaluation order diff --git a/node/web/src/lib.rs b/node/web/src/lib.rs index 2fc576ae1..ef45aab1e 100644 --- a/node/web/src/lib.rs +++ b/node/web/src/lib.rs @@ -54,8 +54,8 @@ pub async fn run(block_producer: Option) -> RpcSender { async fn setup_node( block_producer: Option, ) -> openmina_node_common::Node { - let block_verifier_index = BlockVerifier::make().await; - let work_verifier_index = TransactionVerifier::make().await; + let block_verifier_index = BlockVerifier::make_async().await; + let work_verifier_index = TransactionVerifier::make_async().await; let genesis_config = ::node::config::DEVNET_CONFIG.clone(); let mut node_builder: NodeBuilder = NodeBuilder::new(None, genesis_config); @@ -65,7 +65,7 @@ async fn setup_node( if let Some(bp_key) = block_producer { let provers = - BlockProver::make(Some(block_verifier_index), Some(work_verifier_index)).await; + BlockProver::make_async(Some(block_verifier_index), Some(work_verifier_index)).await; node_builder.block_producer(provers, bp_key); }