From 14e298efa39a6e8982c4b036ea51c6c5b081afd7 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 18:51:48 +0200 Subject: [PATCH 01/17] mina-node-testing: add command to list all available scenarios --- node/testing/src/main.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/node/testing/src/main.rs b/node/testing/src/main.rs index 13c83c3932..02917a3412 100644 --- a/node/testing/src/main.rs +++ b/node/testing/src/main.rs @@ -24,6 +24,7 @@ pub enum Command { ScenariosGenerate(CommandScenariosGenerate), ScenariosRun(CommandScenariosRun), + ScenariosList(CommandScenariosList), } #[derive(Debug, clap::Args)] @@ -57,6 +58,9 @@ pub struct CommandScenariosRun { pub name: String, } +#[derive(Debug, clap::Args)] +pub struct CommandScenariosList {} + impl Command { pub fn run(self) -> Result<(), crate::CommandError> { let rt = setup(); @@ -155,6 +159,13 @@ impl Command { } }) } + Self::ScenariosList(_) => { + println!("Available scenarios:"); + for scenario in Scenarios::iter() { + println!(" {}", scenario.to_str()) + } + Ok(()) + } } } } From 8badb8c5c207e2da2244cb0a7ae9a2f988ad9bb4 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 17:55:58 +0200 Subject: [PATCH 02/17] Website: add testing framework section --- CLAUDE.md | 11 + website/docs/developers/getting-started.mdx | 29 +- .../docs/developers/testing/ledger-tests.md | 160 +++++++ .../developers/testing/ocaml-node-tests.md | 185 ++++++++ website/docs/developers/testing/p2p-tests.md | 205 +++++++++ .../docs/developers/testing/scenario-tests.md | 425 ++++++++++++++++++ .../developers/testing/testing-framework.md | 124 +++++ website/docs/developers/testing/unit-tests.md | 69 +++ website/sidebars.ts | 12 + 9 files changed, 1210 insertions(+), 10 deletions(-) create mode 100644 website/docs/developers/testing/ledger-tests.md create mode 100644 website/docs/developers/testing/ocaml-node-tests.md create mode 100644 website/docs/developers/testing/p2p-tests.md create mode 100644 website/docs/developers/testing/scenario-tests.md create mode 100644 website/docs/developers/testing/testing-framework.md create mode 100644 website/docs/developers/testing/unit-tests.md diff --git a/CLAUDE.md b/CLAUDE.md index 300a7623cf..bcacd97c7e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -234,6 +234,17 @@ import ScriptName from "!!raw-loader!./path/to/script.sh"; This ensures scripts are displayed accurately and stay in sync with the actual files. +### Documentation Style Guidelines + +**Capitalization in headings and bullet points:** + +- Use lowercase for section headings (e.g., "Test design", "Resource + management") +- Use lowercase for bullet point labels (e.g., "**Connection policies**", + "**State inspection**") +- Maintain proper capitalization for proper nouns and technical terms +- Apply this style consistently across all documentation files + ### Documentation Script Testing When modifying developer setup scripts in `website/docs/developers/scripts/`, diff --git a/website/docs/developers/getting-started.mdx b/website/docs/developers/getting-started.mdx index 580451256a..fc481886a5 100644 --- a/website/docs/developers/getting-started.mdx +++ b/website/docs/developers/getting-started.mdx @@ -225,7 +225,7 @@ make postgres-setup make run-archive ``` -### Development Server +### Building documentation For web development with the frontend: @@ -261,28 +261,37 @@ npm install npm start ``` -### Testing Framework +### Testing -The Mina Rust Node includes a comprehensive testing framework: +The Mina Rust Node includes a comprehensive testing framework with unit tests, +integration tests, and scenario-based testing. For detailed information on +running specific types of tests, see the +[Testing Framework](testing/testing-framework) documentation. + +Quick test commands: ```bash -# Build testing binary -make build-testing +# Run basic unit tests +make test -# Run specific test scenarios -./target/release/mina-node-testing --help +# Verify your setup works +make test-p2p ``` ## Next Steps 1. **Explore the Codebase**: Start with [Architecture Documentation](./architecture.md) -2. **Run Tests**: Ensure your setup works by running `make test` -3. **Join the Community**: Connect with other developers on +2. **Learn the Testing Framework**: Familiarize yourself with our comprehensive + [Testing Framework](testing/testing-framework) including unit tests, scenario + tests, and debugging tools +3. **Run Tests**: Ensure your setup works by running tests as described in the + testing documentation +4. **Join the Community**: Connect with other developers on [Discord](https://discord.com/channels/484437221055922177/1290662938734231552) and participate in discussions on the [Mina Protocol Forums](https://forums.minaprotocol.com/) -4. **Read CLAUDE.md**: Check the project-specific development guide in the root +5. **Read CLAUDE.md**: Check the project-specific development guide in the root directory ## Getting Help diff --git a/website/docs/developers/testing/ledger-tests.md b/website/docs/developers/testing/ledger-tests.md new file mode 100644 index 0000000000..a59d14ae74 --- /dev/null +++ b/website/docs/developers/testing/ledger-tests.md @@ -0,0 +1,160 @@ +--- +sidebar_position: 3 +title: Ledger Tests +description: Comprehensive ledger functionality testing +slug: /developers/testing/ledger-tests +--- + +# Ledger Tests + +## Overview + +Comprehensive ledger functionality testing covers transaction processing, state +management, and ledger operations. These tests require nightly Rust due to +advanced features and dependencies. + +## Running Ledger Tests + +### Basic Ledger Testing + +```bash +# Build ledger tests +make build-ledger + +# Run ledger tests +make test-ledger + +# Run with cargo-nextest +make nextest-ledger +``` + +**Note**: Ledger tests require nightly Rust toolchain. + +## VRF Tests + +```bash +# Run VRF tests +make test-vrf + +# Run with cargo-nextest +make nextest-vrf +``` + +**Note**: VRF tests also require nightly Rust toolchain. + +## Block Replayer Tool + +### Purpose + +The block replayer validates transaction and ledger logic by replaying +historical blockchain data: + +- **Transaction Validation**: Verify transaction processing correctness +- **Ledger State**: Ensure ledger state transitions are accurate +- **Protocol Compliance**: Check adherence to protocol rules +- **Performance Analysis**: Measure processing speeds and resource usage + +### Usage + +```bash +# Replay blocks from chain data +./target/release/mina replay --blocks-file chain.json + +# Validate specific block range +./target/release/mina replay --start-block 1000 --end-block 2000 +``` + +### Future development + +Continued development of the block replayer is recommended for: + +- **Enhanced validation**: More comprehensive transaction validation +- **Performance testing**: Benchmark ledger operation performance +- **Protocol evolution**: Test new protocol features against historical data +- **Regression testing**: Ensure changes don't break existing functionality + +## Ledger testing components + +### Transaction processing + +Tests cover: + +- **User commands**: Payment and delegation transactions +- **Fee transfers**: Block producer rewards and fees +- **Coinbase**: Block reward distribution +- **Account creation**: New account initialization + +### State management + +Validation includes: + +- **Account updates**: Balance and nonce changes +- **Merkle tree operations**: Ledger tree consistency +- **State transitions**: Valid state progression +- **Rollback handling**: Proper state restoration + +### Scan state testing + +The scan state manages pending transactions: + +- **Transaction queuing**: Proper ordering and batching +- **SNARK work integration**: Proof generation coordination +- **Staged ledger**: Intermediate state management +- **Fork resolution**: Handling competing chains + +## Performance Considerations + +### Resource management + +1. **Memory usage**: Ledger tests can be memory intensive +2. **Disk I/O**: State persistence operations +3. **CPU usage**: Cryptographic operations and proof verification +4. **Test duration**: Comprehensive tests may take significant time + +### Optimization strategies + +1. **Parallel execution**: Use cargo-nextest for faster test runs +2. **Test filtering**: Run specific test subsets during development +3. **Mock services**: Use lightweight mocks where appropriate +4. **Resource cleanup**: Ensure proper cleanup after tests + +## Best practices + +### Test design + +1. **Isolation**: Use fresh ledger state for each test +2. **Deterministic**: Tests should produce consistent results +3. **Coverage**: Test both success and failure scenarios +4. **Edge cases**: Include boundary conditions and error cases + +### Debugging ledger tests + +1. **State inspection**: Examine ledger state at test failure points +2. **Transaction tracing**: Track transaction processing steps +3. **Merkle tree validation**: Verify tree consistency +4. **Account state**: Check individual account changes + +## Integration with other components + +### Connection to P2P tests + +Ledger tests integrate with P2P testing for: + +- **Transaction propagation**: Ensuring transactions spread correctly +- **Block validation**: Verifying received blocks +- **Consensus integration**: Coordinating with consensus mechanisms + +### Scenario test integration + +Ledger functionality is tested within broader scenarios: + +- **Multi-node synchronization**: Consistent ledger state across nodes +- **Fork resolution**: Proper handling of competing ledgers +- **Performance testing**: Ledger operations under load + +## Related Documentation + +- [Testing Framework Overview](testing-framework): Main testing documentation +- [Unit Tests](unit-tests): Basic component testing +- [Scenario Tests](scenario-tests): Integration testing scenarios +- [Architecture](../architecture): Overall system architecture including ledger diff --git a/website/docs/developers/testing/ocaml-node-tests.md b/website/docs/developers/testing/ocaml-node-tests.md new file mode 100644 index 0000000000..b257f5ddd3 --- /dev/null +++ b/website/docs/developers/testing/ocaml-node-tests.md @@ -0,0 +1,185 @@ +--- +sidebar_position: 5 +title: OCaml Node Tests +description: + Testing OCaml interoperability and cross-implementation compatibility +slug: /developers/testing/ocaml-node-tests +--- + +# OCaml Node Tests + +## Overview + +OCaml node testing focuses on interoperability between the Rust implementation +and the original OCaml Mina node. These tests ensure protocol compliance and +verify that both implementations can work together in the same network. + +## Capabilities with OCaml nodes + +### Basic integration + +- **Multi-node scenarios**: OCaml nodes can participate in complex scenarios +- **Network participation**: Full participation in blockchain network +- **Message exchange**: Proper communication with Rust nodes +- **Block processing**: Handling blocks from both implementations + +### Protocol compliance + +- **Consensus participation**: Following same consensus rules +- **Transaction processing**: Identical transaction handling +- **P2P communication**: Compatible networking protocols +- **State synchronization**: Maintaining consistent blockchain state + +### Network interoperability + +Test communication between implementations: + +```rust +// Mix Rust and OCaml nodes in same test +let mut cluster = Cluster::new(config); +cluster.add_rust_node(rust_config); +cluster.add_ocaml_node(ocaml_config); +cluster.test_interoperability().await; +``` + +## Limitations with OCaml nodes + +### Reduced control capabilities + +- **Deterministic time**: Less precise time control compared to Rust nodes +- **Internal state**: Limited visibility into OCaml node internals +- **Action control**: Restricted ability to trigger specific OCaml actions +- **Debugging**: Fewer debugging and introspection capabilities + +### Testing constraints + +- **State inspection**: Cannot easily examine internal OCaml state +- **Action triggering**: Limited ability to programmatically trigger actions +- **Time synchronization**: Harder to coordinate timing between nodes +- **Error injection**: Difficult to inject specific failure conditions + +## Recommended testing approaches + +### Use case separation + +1. **Use Rust nodes for detailed testing**: + - Complex scenarios and invariant checking + - Precise timing control and state inspection + - Detailed action monitoring and debugging + - Performance analysis and optimization + +2. **Use OCaml nodes for compatibility**: + - Interoperability verification + - Protocol compliance testing + - Real-world compatibility scenarios + - Regression testing against reference implementation + +3. **Mixed scenarios for comprehensive testing**: + - Combine both for integration testing + - Verify cross-implementation communication + - Test network with heterogeneous nodes + - Validate protocol evolution compatibility + +### Testing strategies + +#### Interoperability testing + +Focus on communication between implementations: + +- **Message compatibility**: Ensure messages are understood by both +- **Block acceptance**: Verify blocks from one are accepted by the other +- **Transaction propagation**: Test transaction spread across implementations +- **Consensus participation**: Both implementations follow same consensus + +#### Protocol compliance testing + +Verify both implementations follow the protocol: + +- **State transitions**: Same state changes for same inputs +- **Validation rules**: Identical transaction and block validation +- **Network behavior**: Compatible P2P networking behavior +- **Upgrade compatibility**: Handling protocol version changes + +#### Regression testing + +Ensure changes don't break compatibility: + +- **Backward compatibility**: New Rust changes work with existing OCaml +- **Forward compatibility**: OCaml changes work with Rust implementation +- **Cross-version testing**: Different versions interoperating +- **Protocol evolution**: Smooth transitions between protocol versions + +## Integration Test Scenarios + +### Cross-implementation scenarios + +#### Basic interoperability + +- **Two-node network**: One Rust, one OCaml node +- **Block production**: Each implementation producing blocks +- **Transaction exchange**: Transactions flowing between implementations +- **State consistency**: Both nodes maintaining same state + +#### Multi-node networks + +- **Mixed networks**: Multiple nodes of each implementation +- **Peer discovery**: Implementations discovering each other +- **Message propagation**: Information spreading through mixed network +- **Load balancing**: Work distribution across implementations + +#### Protocol upgrade scenarios + +- **Version migration**: Upgrading one implementation at a time +- **Compatibility windows**: Maintaining interoperability during upgrades +- **Feature rollout**: New features with backward compatibility +- **Deprecation handling**: Removing old protocol features gracefully + +## Testing best practices + +### Scenario design for OCaml integration + +1. **Start with simple scenarios**: Basic two-node interoperability first +2. **Test core protocols**: Focus on essential protocol compliance +3. **Use reference behaviors**: OCaml implementation as reference standard +4. **Handle timing differences**: Account for different timing characteristics + +### Debugging cross-implementation issues + +1. **Compare behaviors**: Side-by-side behavior comparison +2. **Message analysis**: Deep dive into protocol message differences +3. **State divergence**: Identify where states start differing +4. **Version alignment**: Ensure compatible protocol versions + +### Maintenance considerations + +1. **Regular compatibility testing**: Frequent interoperability verification +2. **Protocol change impact**: Assess cross-implementation effects +3. **Performance parity**: Ensure similar performance characteristics +4. **Documentation sync**: Keep compatibility documentation current + +## Future development + +### Enhanced OCaml integration + +Potential improvements for better OCaml testing: + +- **Better state visibility**: Tools for OCaml node state inspection +- **Improved control**: More precise control over OCaml node actions +- **Timing coordination**: Better time synchronization between implementations +- **Debug integration**: Enhanced debugging capabilities for mixed scenarios + +### Automated compatibility testing + +- **Continuous integration**: Regular compatibility test runs +- **Version matrix testing**: Testing multiple version combinations +- **Performance benchmarking**: Comparing implementation performance +- **Compatibility reporting**: Automated compatibility status reporting + +## Related Documentation + +- [Testing Framework Overview](testing-framework): Main testing documentation +- [Scenario Tests](scenario-tests): Integration testing scenarios that can + include OCaml nodes +- [P2P Tests](p2p-tests): P2P networking tests including OCaml compatibility +- [Architecture](../architecture): Understanding the overall system both + implementations share diff --git a/website/docs/developers/testing/p2p-tests.md b/website/docs/developers/testing/p2p-tests.md new file mode 100644 index 0000000000..e54ea8a744 --- /dev/null +++ b/website/docs/developers/testing/p2p-tests.md @@ -0,0 +1,205 @@ +--- +sidebar_position: 4 +title: P2P Tests +description: Peer-to-peer networking tests +slug: /developers/testing/p2p-tests +--- + +# P2P Tests + +## Overview + +Peer-to-peer networking tests validate the networking layer functionality, +including connection establishment, message routing, and network resilience. + +## Running P2P Tests + +### Basic P2P Testing + +```bash +# Run P2P tests +make test-p2p + +# Run with cargo-nextest +make nextest-p2p +``` + +## P2P Testing Scenarios + +### Connection establishment + +Low-level networking testing covers: + +- **Basic peer connections**: Establishing peer-to-peer connections +- **Handshake protocol**: Connection initialization and authentication +- **Transport layer**: Both libp2p and WebRTC transport validation +- **Peer discovery**: Finding and connecting to network peers + +### Message routing + +Message handling and propagation: + +- **Message delivery**: Proper message routing between peers +- **Gossip protocols**: Message propagation through the network +- **Request/response**: Direct peer-to-peer communication +- **Message prioritization**: Handling different message types + +### Network resilience + +Testing network fault tolerance: + +- **Network partitions**: Handling network splits and merges +- **Connection recovery**: Reconnection after temporary failures +- **Peer churn**: Handling peers joining and leaving the network +- **Load testing**: Performance under high message volume + +## Dual Transport Architecture + +The Mina Rust node supports dual transport layers: + +### libp2p transport + +Traditional blockchain networking: + +- **TCP/IP based**: Standard internet protocols +- **DHT integration**: Distributed hash table for peer discovery +- **NAT traversal**: Connection through firewalls and NATs +- **Protocol multiplexing**: Multiple protocols over single connection + +### WebRTC transport + +Browser-compatible networking: + +- **UDP based**: Lower latency communication +- **Browser compatibility**: Direct browser-to-node connections +- **ICE protocol**: Interactive Connectivity Establishment +- **STUN/TURN**: NAT traversal support + +## P2P testing components + +### Connection management + +- **Peer pool**: Managing active connections +- **Connection limits**: Respecting maximum connection counts +- **Bandwidth management**: Controlling data flow rates +- **Connection quality**: Monitoring connection health + +### Message types + +Different message categories tested: + +- **Block messages**: New block announcements and requests +- **Transaction messages**: Transaction propagation +- **Consensus messages**: Consensus-related communication +- **Peer messages**: Peer discovery and management + +### Network debugger integration + +P2P tests integrate with the network debugger for: + +- **Connection inspection**: Real-time connection monitoring +- **Message tracing**: Following messages through the network +- **Performance metrics**: Bandwidth and latency measurements +- **Failure analysis**: Debugging connection issues + +## Advanced P2P Features + +### Custom channel abstractions + +The framework provides channel abstractions for different message types: + +```rust +// Example: Message channel usage +let block_channel = p2p.get_channel::(); +block_channel.broadcast(new_block).await; + +let response = p2p.request::(peer_id, request).await; +``` + +### Peer discovery mechanisms + +Multiple discovery methods: + +- **Bootstrap nodes**: Initial connection points +- **DHT discovery**: Distributed peer discovery +- **mDNS discovery**: Local network discovery +- **Gossip discovery**: Learning peers through existing connections + +### Network policies + +Configurable networking behavior: + +- **Connection policies**: Which peers to connect to +- **Message filtering**: What messages to accept/forward +- **Rate limiting**: Preventing spam and DoS attacks +- **Reputation systems**: Peer quality scoring + +## WebRTC Specific Testing + +### Signaling tests + +WebRTC requires signaling for connection establishment: + +- **Offer/answer exchange**: SDP negotiation +- **ICE candidate exchange**: Connection path discovery +- **Signaling server**: Central coordination point +- **Browser integration**: Testing browser-based connections + +### NAT traversal + +WebRTC's NAT traversal capabilities: + +- **STUN server**: NAT type detection +- **TURN server**: Relay connections when direct fails +- **ICE gathering**: Finding connection paths +- **Connection fallback**: Graceful degradation + +## Performance testing + +### Throughput testing + +Measuring network performance: + +- **Message throughput**: Messages per second +- **Bandwidth usage**: Data transfer rates +- **Latency measurement**: Message delivery times +- **Connection scaling**: Performance with many peers + +### Load testing + +Testing under stress: + +- **High peer count**: Many simultaneous connections +- **Message flooding**: High message rates +- **Resource usage**: Memory and CPU under load +- **Degradation patterns**: How performance degrades + +## Best practices + +### Test design + +1. **Network isolation**: Use isolated test networks +2. **Deterministic tests**: Control network timing where possible +3. **Error injection**: Test failure scenarios +4. **Resource cleanup**: Properly close connections + +### Debugging P2P issues + +1. **Network debugger**: Use sidecar for connection inspection +2. **Connection logs**: Monitor connection establishment +3. **Message tracing**: Follow message paths +4. **Performance metrics**: Monitor bandwidth and latency + +### Integration considerations + +1. **Scenario integration**: P2P tests within broader scenarios +2. **Multi-transport**: Test both libp2p and WebRTC +3. **Cross-implementation**: Test with OCaml nodes +4. **Real network conditions**: Test with actual internet conditions + +## Related Documentation + +- [Testing Framework Overview](testing-framework): Main testing documentation +- [Scenario Tests](scenario-tests): Integration testing scenarios +- [P2P Networking](../p2p-networking): P2P architecture details +- [WebRTC](../webrtc): WebRTC implementation details diff --git a/website/docs/developers/testing/scenario-tests.md b/website/docs/developers/testing/scenario-tests.md new file mode 100644 index 0000000000..d01f5a1d81 --- /dev/null +++ b/website/docs/developers/testing/scenario-tests.md @@ -0,0 +1,425 @@ +--- +sidebar_position: 2 +title: Scenario Tests +description: + Deterministic scenario-based testing using the mina-node-testing framework +slug: /developers/testing/scenario-tests +--- + +# Scenario Tests + +## Overview + +Scenario tests provide deterministic, scenario-based testing for complex +multi-node blockchain scenarios using the `mina-node-testing` framework. Tests +are structured as sequences of steps that can be recorded, saved, and replayed +deterministically across different environments. + +## Architecture + +### Core design principles + +1. **Scenario-based testing**: Tests are structured as scenarios consisting of + ordered sequences of steps that can be recorded, saved, and replayed + deterministically +2. **State machine architecture**: Follows the Redux-style pattern used + throughout the Mina Rust node for predictable state transitions +3. **Multi-implementation support**: Tests both Rust (the Mina Rust node) and + OCaml (original Mina) nodes in the same scenarios +4. **Deterministic replay**: All tests can be replayed exactly using recorded + scenarios with fixed random seeds and controlled time progression + +### mina-node-testing framework + +#### Test library (`node/testing/src/`) + +The `mina-node-testing` library provides the core runtime infrastructure: + +- **Cluster management**: Coordinates multiple node instances with precise + lifecycle control +- **Scenario orchestration**: Manages scenario execution, step ordering, and + synchronization +- **State monitoring**: Observes node state changes and validates transitions +- **Action dispatch**: Triggers actions across nodes with deterministic timing +- **Recording/replay**: Captures and reproduces test scenarios with complete + fidelity + +#### Test runner (`node/testing/src/bin/runner.rs`) + +The test runner provides comprehensive scenario management: + +- **Scenario execution**: Runs scenarios from templates or recorded files +- **Step-by-step control**: Allows precise control over scenario progression +- **Progress monitoring**: Tracks scenario execution progress with detailed + logging +- **Error handling**: Captures and reports test failures with full context +- **Replay capability**: Reproduces exact scenarios for debugging + +#### Scenario structure + +Scenarios consist of two main components: + +- **ScenarioInfo**: Metadata including scenario name, description, and + configuration +- **ScenarioSteps**: Ordered sequence of actions to be executed + +Common step types include: + +- **Node management**: Adding, removing, and configuring nodes +- **Network operations**: Connecting nodes, establishing P2P connections +- **Time control**: Advancing time deterministically for reproducible tests +- **Event dispatch**: Triggering specific actions on nodes +- **Validation**: Checking conditions and timeouts +- **State assertions**: Verifying expected node states + +## Testing capabilities + +### Cluster management + +The framework provides comprehensive cluster orchestration: + +```rust +use mina_node_testing::scenario::{Scenario, ScenarioStep}; + +// Create cluster with deterministic configuration +let scenario = Scenario::builder() + .with_info("multi_node_sync", "Test multi-node synchronization") + .with_step(ScenarioStep::AddNode { + node_id: "node1".to_string(), + config: NodeConfig::rust_default() + }) + .with_step(ScenarioStep::AddNode { + node_id: "node2".to_string(), + config: NodeConfig::rust_default() + }) + .with_step(ScenarioStep::ConnectNodes { + dialer: "node1".to_string(), + listener: "node2".to_string() + }) + .build(); +``` + +### Deterministic time control + +Precise time control ensures reproducible test outcomes: + +```rust +// Advance time in controlled increments +scenario.add_step(ScenarioStep::AdvanceTime { + duration: Duration::from_secs(30) +}); + +// Check conditions with deterministic timeouts +scenario.add_step(ScenarioStep::CheckTimeout { + timeout: Duration::from_secs(60), + condition: "nodes_synchronized".to_string() +}); +``` + +### Cross-implementation testing + +Seamless integration of Rust and OCaml nodes: + +```rust +// Mix implementations in same scenario +scenario + .with_step(ScenarioStep::AddNode { + node_id: "rust_node".to_string(), + config: NodeConfig::rust_default() + }) + .with_step(ScenarioStep::AddNode { + node_id: "ocaml_node".to_string(), + config: NodeConfig::ocaml_default() + }) + .with_step(ScenarioStep::TestInteroperability { + nodes: vec!["rust_node".to_string(), "ocaml_node".to_string()] + }); +``` + +### Invariant checking system + +Comprehensive invariant validation throughout scenario execution: + +```rust +// Define invariants to check continuously +scenario.add_invariant(Invariant::new( + "consistent_best_tip", + |cluster| cluster.nodes_have_consistent_best_tip(), + InvariantCheckFrequency::EveryStep +)); + +// Custom invariants for specific conditions +scenario.add_invariant(Invariant::new( + "connection_stability", + |cluster| cluster.all_connections_stable(), + InvariantCheckFrequency::AfterTimeAdvancement +)); +``` + +### Scenario inheritance + +Build complex scenarios from simpler components: + +```rust +// Base scenario for node setup +let base_scenario = Scenario::builder() + .with_info("base_two_nodes", "Basic two-node setup") + .with_step(ScenarioStep::AddNode { /* ... */ }) + .with_step(ScenarioStep::AddNode { /* ... */ }) + .build(); + +// Extended scenario inheriting base setup +let extended_scenario = base_scenario + .extend() + .with_info("two_nodes_with_blocks", "Two nodes producing blocks") + .with_step(ScenarioStep::ProduceBlocks { count: 5 }) + .build(); +``` + +## Scenario categories + +### Bootstrap scenarios + +Node initialization and basic functionality: + +- **Single node bootstrap**: Basic node startup and initialization +- **Multi-node bootstrap**: Coordinated startup of multiple nodes +- **State recovery**: Node restart and state restoration +- **Configuration validation**: Testing different node configurations + +### Network formation scenarios + +Peer-to-peer network establishment: + +- **Initial network formation**: Nodes discovering and connecting to peers +- **Peer discovery mechanisms**: Testing different discovery protocols +- **Network topology**: Various network connection patterns +- **Large-scale networks**: Testing with many nodes (10+ nodes) + +### Synchronization scenarios + +Blockchain state synchronization: + +- **Block propagation**: Blocks spreading through the network +- **Fork resolution**: Handling competing blockchain forks +- **Catchup mechanisms**: Nodes synchronizing after downtime +- **State consistency**: Ensuring all nodes reach same state + +### Transaction scenarios + +Transaction processing and propagation: + +- **Transaction pool**: Managing pending transactions +- **Transaction propagation**: Spreading transactions across network +- **Transaction validation**: Ensuring proper transaction processing +- **Mempool management**: Testing transaction pool behavior + +### P2P networking scenarios + +Low-level networking functionality: + +- **Connection establishment**: Basic peer-to-peer connections +- **Message routing**: Proper message delivery and routing +- **Network partitions**: Handling network splits and merges +- **Connection recovery**: Reconnection after network failures +- **Protocol compatibility**: Testing different protocol versions + +### Cross-implementation scenarios + +Interoperability between implementations: + +- **Rust-OCaml interoperability**: Mixed implementation networks +- **Protocol compliance**: Adherence to Mina protocol specifications +- **Message compatibility**: Cross-implementation communication +- **Consensus participation**: Shared consensus across implementations + +## Advanced testing features + +### Proof configuration and mocking + +Control proof generation behavior for faster test execution: + +```rust +use mina_node_testing::config::{ProofConfig, ClusterConfig}; + +// Disable proofs entirely for speed +let config = ClusterConfig::new() + .with_proof_config(ProofConfig::Disabled); + +// Use dummy/mock proofs +let config = ClusterConfig::new() + .with_proof_config(ProofConfig::Dummy); + +// Use real proofs (slower but comprehensive) +let config = ClusterConfig::new() + .with_proof_config(ProofConfig::Real); +``` + +### Random seed control + +Ensure deterministic test execution: + +```rust +// Fixed seed for reproducible randomness +let scenario = Scenario::builder() + .with_random_seed(12345) + .with_info("deterministic_test", "Reproducible scenario"); + +// Different seed for variation testing +let scenario = Scenario::builder() + .with_random_seed(67890) + .with_info("variation_test", "Alternative execution path"); +``` + +### Large-scale network testing + +Test scenarios with many nodes: + +```rust +// Create 10-node network efficiently +let scenario = Scenario::builder() + .with_info("large_network", "10-node network formation") + .with_steps((0..10).map(|i| + ScenarioStep::AddNode { + node_id: format!("node_{}", i), + config: NodeConfig::rust_default() + } + ).collect()) + .with_step(ScenarioStep::ConnectAllNodes) + .with_step(ScenarioStep::WaitForFullMesh); +``` + +### Debugging and introspection + +Advanced debugging capabilities: + +```rust +// Enable detailed logging +scenario.add_step(ScenarioStep::SetLogLevel { + level: "trace".to_string(), + components: vec!["p2p".to_string(), "consensus".to_string()] +}); + +// Take state snapshots +scenario.add_step(ScenarioStep::TakeSnapshot { + name: "before_fork".to_string() +}); + +// Compare states between nodes +scenario.add_step(ScenarioStep::CompareNodeStates { + nodes: vec!["node1".to_string(), "node2".to_string()], + fields: vec!["best_tip".to_string(), "peer_count".to_string()] +}); +``` + +## Running tests + +The list of available tests can be found by running: + +``` +cargo run --release --features scenario-generators,p2p-webrtc \ + --bin mina-node-testing -- scenarios-list +``` + +### Scenario generation and replay + +The `mina-node-testing` framework supports both scenario generation and replay: + +```bash +# Generate specific scenarios (requires scenario-generators feature) +cargo run --release --features scenario-generators --bin mina-node-testing -- \ + scenarios-generate --name record-replay-block-production + +# Generate WebRTC scenarios (requires additional p2p-webrtc feature) +cargo run --release --features scenario-generators,p2p-webrtc \ + --bin mina-node-testing -- scenarios-generate --name p2p-signaling + +# Generate multi-node scenarios +cargo run --release --features scenario-generators --bin mina-node-testing -- \ + scenarios-generate --name multi-node-pubsub-propagate-block + +# Replay existing scenarios (use actual scenario names from list) +cargo run --release --bin mina-node-testing -- scenarios-run --name p2p-signaling +``` + +### Key execution features + +- **Feature-based compilation**: Use `--features` to enable specific test + capabilities +- **Scenario generation**: Create new test scenarios from templates +- **Scenario replay**: Execute previously generated or recorded scenarios +- **Named scenarios**: Reference scenarios by name for consistent execution + +### Environment configuration + +For network connectivity in testing environments, you may need to configure: + +```bash +# Enable connection to replayer service (used in CI) +export REPLAYER_MULTIADDR="/dns4/primary-tcp-proxy.hz.minaprotocol.network/tcp/40110/p2p/12D3KooWPayQEdprqY2m3biReUUybA5LoULpJE7YWu6wetEKKELv" + +# Allow local address discovery +export MINA_DISCOVERY_FILTER_ADDR=false + +# Maintain connections with unknown streams (for replayer compatibility) +export KEEP_CONNECTION_WITH_UNKNOWN_STREAM=true +``` + +The `REPLAYER_MULTIADDR` variable specifies the multiaddress for connecting to +the Mina protocol network's primary TCP proxy, enabling scenario tests to +interact with live network components when needed. + +## Best practices + +### Scenario design principles + +1. **Start simple**: Begin with single-node scenarios before advancing to + multi-node complexity +2. **Use scenario inheritance**: Build complex scenarios from proven simpler + components +3. **Minimize time advancement**: Use smallest time increments necessary for + deterministic behavior +4. **Add comprehensive invariants**: Define and check system properties + throughout test execution +5. **Use fixed random seeds**: Ensure reproducible test outcomes across + environments +6. **Handle resource cleanup**: Properly clean up nodes and network resources + after tests + +### Deterministic testing guidelines + +1. **Control all randomness**: Use fixed seeds for any random operations +2. **Minimize external dependencies**: Avoid relying on external services or + timing +3. **Use precise time control**: Advance time in controlled, minimal increments +4. **Validate state transitions**: Check node states after each significant step +5. **Test failure scenarios**: Include network partitions, node failures, and + edge cases + +### Debugging and troubleshooting + +1. **Enable comprehensive logging**: Use detailed logging levels for failing + scenarios +2. **Use state snapshots**: Capture node state at critical points for comparison +3. **Replay with variations**: Test with different seeds and timing to isolate + issues +4. **Compare node states**: Verify consistency between nodes at checkpoint steps +5. **Validate invariants**: Ensure system properties hold throughout scenario + execution + +### Performance optimization + +1. **Use proof mocking**: Disable or mock proofs during development for faster + iteration +2. **Minimize node count**: Use smallest number of nodes that demonstrate the + behavior +3. **Batch similar operations**: Group related steps together for efficiency +4. **Profile resource usage**: Monitor memory and CPU usage during large + scenarios + +## Related Documentation + +- [Testing Framework Overview](testing-framework): Main testing documentation +- [Unit Tests](unit-tests): Basic component testing +- [P2P Tests](p2p-tests): P2P networking specific tests +- [OCaml Node Tests](ocaml-node-tests): OCaml interoperability testing diff --git a/website/docs/developers/testing/testing-framework.md b/website/docs/developers/testing/testing-framework.md new file mode 100644 index 0000000000..b74d237f2b --- /dev/null +++ b/website/docs/developers/testing/testing-framework.md @@ -0,0 +1,124 @@ +--- +sidebar_position: 1 +title: Testing Framework +description: Comprehensive testing infrastructure for the Mina Rust node +slug: /developers/testing/testing-framework +--- + +# Testing Framework + +## Overview + +The Mina Rust node testing infrastructure provides comprehensive testing +capabilities for blockchain functionality. The framework is designed around +scenario-based testing with deterministic replay capabilities, multi-node +orchestration, and cross-implementation compatibility testing. + +## Testing Architecture + +### Core Design Principles + +1. **Scenario-Based Testing**: Tests are structured as scenarios - sequences of + steps that can be recorded, saved, and replayed deterministically +2. **State Machine Architecture**: Follows the Redux-style pattern used + throughout the Mina Rust node +3. **Multi-Implementation Support**: Tests both Rust (the Mina Rust node) and + OCaml (original Mina) nodes +4. **Deterministic Replay**: All tests can be replayed exactly using recorded + scenarios + +### Component Overview + +The testing framework consists of several specialized testing areas: + +- **[Unit Tests](unit-tests)**: Basic component and function testing +- **[Scenario Tests](scenario-tests)**: Integration testing with network + debugger +- **[Ledger Tests](ledger-tests)**: Comprehensive ledger functionality testing +- **[P2P Tests](p2p-tests)**: Peer-to-peer networking validation +- **[OCaml Node Tests](ocaml-node-tests)**: Cross-implementation compatibility + +## Development Workflow + +### Adding New Tests + +1. **Determine test type**: Choose the appropriate testing category +2. **Follow existing patterns**: Use existing tests as templates +3. **Document purpose**: Clear documentation of what is being tested +4. **Integration**: Ensure tests run in CI/CD pipeline + +### Debugging Test Failures + +1. **Check logs**: Review detailed test output and node logs +2. **Use network debugger**: For scenario and P2P test issues +3. **Reproduce locally**: Run failing tests in local environment +4. **State analysis**: Examine node state when tests fail + +### Performance Considerations + +1. **Resource management**: Monitor memory and CPU usage during tests +2. **Timeout handling**: Set appropriate timeouts for test conditions +3. **Parallel execution**: Use cargo-nextest for faster test execution +4. **Cleanup**: Ensure proper resource cleanup after tests + +## Advanced Features + +### Deterministic Testing + +The framework provides deterministic test execution through: + +- **Time control**: Precise control over time progression in tests +- **State recording**: Capture exact state transitions +- **Replay capability**: Reproduce exact test scenarios +- **Invariant checking**: Continuous validation of system properties + +### Multi-Node Orchestration + +Support for complex multi-node scenarios: + +- **Cluster management**: Coordinate multiple node instances +- **Synchronization**: Ensure proper ordering of operations +- **Network simulation**: Control network conditions and failures +- **Cross-implementation**: Mix Rust and OCaml nodes in same tests + +### Network Debugging + +Integration with network debugger provides: + +- **Connection inspection**: Real-time network monitoring +- **Message tracing**: Follow messages through the network +- **Performance analysis**: Bandwidth and latency measurements +- **Failure diagnosis**: Debug network-related issues + +## Best Practices + +### Test Design + +1. **Start simple**: Begin with unit tests, build up to integration tests +2. **Test incrementally**: Build complex scenarios from simpler components +3. **Use appropriate tools**: Choose right test type for the functionality +4. **Handle edge cases**: Include failure scenarios and boundary conditions + +### Maintenance + +1. **Regular updates**: Keep tests updated with code changes +2. **Performance monitoring**: Track test execution times +3. **Flaky test management**: Identify and fix non-deterministic failures +4. **Documentation**: Maintain current test documentation + +## CI/CD Integration + +Tests are integrated into the continuous integration pipeline: + +- **Automated execution**: Tests run on every pull request +- **Multiple platforms**: Testing across different operating systems +- **Container isolation**: Each test runs in clean environment +- **Artifact collection**: Test results and logs archived for analysis + +## Related Documentation + +- [Getting Started](../getting-started): Basic development setup including + testing +- [Architecture](../architecture): Overall system architecture that tests + validate +- [P2P Networking](../p2p-networking): Network protocols tested by P2P scenarios diff --git a/website/docs/developers/testing/unit-tests.md b/website/docs/developers/testing/unit-tests.md new file mode 100644 index 0000000000..cf236a556e --- /dev/null +++ b/website/docs/developers/testing/unit-tests.md @@ -0,0 +1,69 @@ +--- +sidebar_position: 1 +title: Unit Tests +description: Basic unit tests for individual components +slug: /developers/testing/unit-tests +--- + +# Unit Tests + +## Overview + +Basic unit tests for individual components provide the foundation of the testing +infrastructure. These tests focus on testing individual functions, modules, and +components in isolation. + +## Running Unit Tests + +### Standard Unit Tests + +Run all unit tests across the codebase: + +```bash +# Run all unit tests +make test + +# Run tests in release mode +make test-release + +# Use cargo-nextest for faster execution +make nextest +``` + +### Performance Considerations + +1. **Resource management**: Monitor memory and CPU usage during tests +2. **Timeout handling**: Set appropriate timeouts for test conditions +3. **Parallel execution**: Run independent tests concurrently +4. **Cleanup**: Ensure proper resource cleanup after tests + +## Best Practices + +### Test Design + +1. **Isolation**: Ensure tests don't depend on external state +2. **Deterministic**: Tests should produce consistent results +3. **Fast**: Unit tests should execute quickly +4. **Clear**: Test names should clearly describe what is being tested + +### Debugging Unit Tests + +1. **Enable logging**: Use detailed logging for test debugging +2. **Check assumptions**: Ensure test assumptions match actual behavior +3. **Reproduce locally**: Run failing tests locally for easier debugging +4. **Update expectations**: Adjust test expectations if behavior changed + +## Integration with CI + +Unit tests run automatically in CI with: + +- **Container isolation**: Each test runs in clean container +- **Parallel execution**: Multiple tests run concurrently +- **Artifact collection**: Test results archived for analysis + +## Related Documentation + +- [Testing Framework Overview](testing-framework): Main testing documentation +- [Scenario Tests](scenario-tests): Integration testing scenarios +- [Getting Started](../getting-started): Basic development setup including + testing diff --git a/website/sidebars.ts b/website/sidebars.ts index 101bddf9e8..f8d9a746bd 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -85,6 +85,18 @@ const sidebars: SidebarsConfig = { 'developers/libp2p', ], }, + { + type: 'category', + label: 'Testing', + items: [ + 'developers/testing/testing-framework', + 'developers/testing/unit-tests', + 'developers/testing/scenario-tests', + 'developers/testing/ledger-tests', + 'developers/testing/p2p-tests', + 'developers/testing/ocaml-node-tests', + ], + }, { type: 'category', label: 'Future Work', From 418eb95f60c3df3ed28f2db4f372ce187cb2c79c Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 19:40:35 +0200 Subject: [PATCH 03/17] Node/testing: openmina -> Mina Rust --- node/testing/src/scenarios/solo_node/bootstrap.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/testing/src/scenarios/solo_node/bootstrap.rs b/node/testing/src/scenarios/solo_node/bootstrap.rs index d30b523ffd..d1a4833e7b 100644 --- a/node/testing/src/scenarios/solo_node/bootstrap.rs +++ b/node/testing/src/scenarios/solo_node/bootstrap.rs @@ -52,7 +52,7 @@ impl SoloNodeBootstrap { let node_id = runner.add_rust_node(config.initial_peers(vec![ListenerNode::Custom(replayer)])); - eprintln!("launch Openmina node with default configuration, id: {node_id}"); + eprintln!("launch Mina Rust node with default configuration, id: {node_id}"); let mut timeout = TIMEOUT; let mut last_time = Instant::now(); From 72cb9e753599c658cb25899aa7ec203622906b84 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 21:06:11 +0200 Subject: [PATCH 04/17] Website: move scenarios/readme to the website --- .../testing/network-connectivity.md | 87 +++++++++++-------- website/sidebars.ts | 1 + 2 files changed, 51 insertions(+), 37 deletions(-) rename node/testing/src/scenarios/readme.md => website/docs/developers/testing/network-connectivity.md (79%) diff --git a/node/testing/src/scenarios/readme.md b/website/docs/developers/testing/network-connectivity.md similarity index 79% rename from node/testing/src/scenarios/readme.md rename to website/docs/developers/testing/network-connectivity.md index 09a8522b92..df2389ba3d 100644 --- a/node/testing/src/scenarios/readme.md +++ b/website/docs/developers/testing/network-connectivity.md @@ -1,3 +1,10 @@ +--- +sidebar_position: 5 +title: Network Connectivity Testing +description: Detailed network connectivity and peer management testing scenarios +slug: /developers/testing/network-connectivity +--- + # Network Connectivity and Peer Management ## Network Connectivity @@ -7,7 +14,7 @@ synchronize with the network. _This test assesses the blockchain node's ability to maintain consistent network connectivity. It evaluates whether a node can gracefully handle temporary -disconnections from the network and subsequently reestablish connections. _ +disconnections from the network and subsequently reestablish connections._ We want to ensure that new nodes can join the network and handle being overwhelmed with connections or data requests, including various resilience and @@ -19,19 +26,19 @@ participate in the blockchain's consensus process. We are testing two versions of the node: -### Solo node +## Solo node We want to be able to test whether the Rust node is compatible with the OCaml -node. We achieve this by attempting to connect the Must Rust node to the -existing OCaml testnet. +node. We achieve this by attempting to connect the Rust node to the existing +OCaml testnet. -For that purpose, we are utilizing a _solo node_, which is a single Open Mina -node connected to a network of OCaml nodes. Currently, we are using the public +For that purpose, we are utilizing a _solo node_, which is a single Rust node +connected to a network of OCaml nodes. Currently, we are using the public testnet, but later on we want to use our own network of OCaml nodes on our cluster. -This test is performed by launching an Must Rust node and connecting it to seed -nodes of the public (or private) OCaml testnet. +This test is performed by launching a Rust node and connecting it to seed nodes +of the public (or private) OCaml testnet. _The source code for this test can be found in this repo:_ @@ -39,21 +46,21 @@ _The source code for this test can be found in this repo:_ We are testing these scenarios: -1. Whether the Must Rust node can accept an incoming connection from OCaml node. - This test will prove our Must Rust node is listening to incoming connections - and can accept them. -2. Whether the OCaml node can discover and connect to an Must Rust node that is - advertising itself. This is done by advertising the Must Rust node so that - the OCaml node can discover it and connect to the node. +1. Whether the Rust node can accept an incoming connection from OCaml node. This + test will prove our Rust node is listening to incoming connections and can + accept them. +2. Whether the OCaml node can discover and connect to a Rust node that is + advertising itself. This is done by advertising the Rust node so that the + OCaml node can discover it and connect to the node. This test is the same as the previous one, except we do not inform the OCaml node to connect to it explicitly, it should find it automatically and connect using peer discovery (performed through Kademlia). This test will ensure the - Must Rust node uses Kademlia in a way that is compatible with the OCaml node. + Rust node uses Kademlia in a way that is compatible with the OCaml node. However, with this test, we are currently experiencing problems that may be -caused by OCaml nodes not being currently able to "see" the Must Rust nodes, -because our implementation of the p2p layer is incomplete. +caused by OCaml nodes not being currently able to "see" the Rust nodes, because +our implementation of the p2p layer is incomplete. We have implemented the missing protocol (Kademlia) into the p2p layer to make OCaml nodes see our node. Despite being successfully implemented, the main test @@ -64,7 +71,7 @@ We are also missing certain p2p protocols like `/mina/peer-exchange`, `/mina/bitswap-exchange`, `/mina/node-status`, `/ipfs/id/1.0.0` While these p2p protocol may not be relevant, it is possible OCaml nodes do not -recognize the Must Rust node because we are missing some of them. +recognize the Rust node because we are missing some of them. We run these tests until: @@ -74,11 +81,10 @@ We run these tests until: - The test is failed if the specified number of steps occur but the conditions are not met. -#### Kademlia peer discovery +### Kademlia peer discovery -We want the Open Mina node to be able to connect to peers, both other Open Mina -nodes (that are written in Rust) as well as native Mina nodes (written in -OCaml). +We want the Rust node to be able to connect to peers, both other Rust nodes as +well as native Mina nodes (written in OCaml). Native Mina nodes use Kademlia (KAD), a distributed hash table (DHT) for peer-to-peer computer networks. Hash tables are data structures that map _keys_ @@ -91,10 +97,10 @@ the network. Since we initially focused on other parts of the node, we used the RPC get_initial_peers as a sort-of workaround to connect our nodes between -themselves. Now, to ensure compatibility with the native Mina node, we’ve -implemented KAD for peer discovery for the Open Mina node. +themselves. Now, to ensure compatibility with the native Mina node, we've +implemented KAD for peer discovery for the Rust node. -#### How does Mina utilize Kademlia? +### How does Mina utilize Kademlia? Kademlia has two main parts - the routing table and the peer store. @@ -114,14 +120,14 @@ messages. A provider in Kademlia announces possession of specific data (identified by a unique key) and shares it with others. In MINA's case, all providers use the same key, which is the SHA256 hash of a specific string pattern. In MINA, every -node acts as a “provider,” making the advertisement as providers redundant. +node acts as a "provider," making the advertisement as providers redundant. Non-network nodes are filtered at the PNet layer. If there are no peers, KAD will automatically search for new ones. KAD will also search for new peers whenever the node is restarted. If a connection is already made, it will search for more peers every hour. -#### Message types +### Message types - AddProvider - informs the peer that you can provide the information described by the specified key. @@ -130,10 +136,10 @@ made, it will search for more peers every hour. where your node should be. Or it may find a node that you need to send an AddProvider (or GetProviders) message to. -#### Potential issues identified +### Potential issues identified -- An earlier issue in the Open Mina (Rust node) with incorrect provider key - advertising is now fixed. +- An earlier issue in the Rust node with incorrect provider key advertising is + now fixed. - The protocol's use in OCaml nodes might be a potential security risk; an adversary could exploit this to DoS the network. One possible solution is to treat all Kademlia peers as providers. @@ -146,7 +152,7 @@ made, it will search for more peers every hour. - The malicious peer can deliberately choose the peer_id and deny access to the information, just always say there are no providers. This problem has been inherited from the OCaml implementation of the node. We have mitigated it by - making the Must Rust node not rely on GetProviders, instead, we only do + making the Rust node not rely on GetProviders, instead, we only do AddProviders to advertise ourselves, but treat any peer of the Kademlia network as a valid Mina peer, no matter if it is a provider or not, so a malicious peer can prevent OCaml nodes from discovering us, but it will not @@ -162,12 +168,12 @@ made, it will search for more peers every hour. - We need more testing on support for IPv6. libp2p_helper code can't handle IPv6 for the IP range filtering. -### Multi node +## Multi node -We also want to test a scenario in which the network consists only of Must Rust -nodes. If the Must Rust node is using a functionality that is implemented only -in the OCaml node, and it does not perform it correctly, then we will not be -able to see it with solo node test. +We also want to test a scenario in which the network consists only of Rust +nodes. If the Rust node is using a functionality that is implemented only in the +OCaml node, and it does not perform it correctly, then we will not be able to +see it with solo node test. For that purpose, we utilize a Multi node test, which involves a network of our nodes, without any third party, so that the testing is completely local and @@ -177,7 +183,7 @@ _The source code for this test can be found in this repo:_ [https://github.com/o1-labs/mina-rust/blob/develop/node/testing/src/scenarios/multi_node/basic_connectivity_initial_joining.rs#L9](https://github.com/o1-labs/mina-rust/blob/develop/node/testing/src/scenarios/multi_node/basic_connectivity_initial_joining.rs#L9) -#### How it's tested +### How it's tested **Node cluster**: We use a `ClusterRunner` utility to manage the setup and execution of test scenarios on a cluster of nodes. @@ -212,3 +218,10 @@ scenarios, ensuring parent scenarios are run before their children. asynchronous method for setting up a cluster according to a specified configuration and running all parent scenarios to prepare the environment for a specific test. + +## Related Documentation + +- [Testing Framework Overview](testing-framework): Main testing documentation +- [Scenario Tests](scenario-tests): Scenario-based testing framework +- [P2P Tests](p2p-tests): P2P networking specific tests +- [OCaml Node Tests](ocaml-node-tests): OCaml interoperability testing diff --git a/website/sidebars.ts b/website/sidebars.ts index f8d9a746bd..a42b956082 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -94,6 +94,7 @@ const sidebars: SidebarsConfig = { 'developers/testing/scenario-tests', 'developers/testing/ledger-tests', 'developers/testing/p2p-tests', + 'developers/testing/network-connectivity', 'developers/testing/ocaml-node-tests', ], }, From ba022c6e088c08147c5be6d73c8af42bcf0a840d Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 21:16:25 +0200 Subject: [PATCH 05/17] Node/testing: add top level docs to module linked to the website --- node/testing/src/lib.rs | 33 ++++++++++++++++++++++ node/testing/src/main.rs | 24 ++++++++++++++++ node/testing/src/scenarios/mod.rs | 46 ++++++++++++++++++++++++++----- 3 files changed, 96 insertions(+), 7 deletions(-) diff --git a/node/testing/src/lib.rs b/node/testing/src/lib.rs index 81a5ee7cf0..ddeb360732 100644 --- a/node/testing/src/lib.rs +++ b/node/testing/src/lib.rs @@ -1,3 +1,36 @@ +//! # Mina Node Testing Framework +//! +//! A comprehensive testing framework for the Mina Rust node implementation. +//! Provides scenario-based testing capabilities with deterministic execution, +//! cluster management, and cross-implementation compatibility testing. +//! +//! ## Key Features +//! +//! - **Scenario-based testing**: Deterministic, repeatable test sequences +//! - **Cluster orchestration**: Multi-node test coordination +//! - **Cross-implementation**: Tests both Rust and OCaml node compatibility +//! - **Recording/replay**: Capture and reproduce test scenarios +//! - **Network simulation**: Controlled network environments +//! +//! ## Documentation +//! +//! For detailed usage and examples, see: +//! - [Scenario Tests](https://o1-labs.github.io/mina-rust/developers/testing/scenario-tests) +//! - [Testing Framework](https://o1-labs.github.io/mina-rust/developers/testing/testing-framework) +//! - [Network Connectivity](https://o1-labs.github.io/mina-rust/developers/testing/network-connectivity) +//! +//! ## Usage +//! +//! The main entry point is the `mina-node-testing` binary: +//! +//! ```bash +//! # List available scenarios +//! cargo run --release --bin mina-node-testing -- scenarios-list +//! +//! # Run a specific scenario +//! cargo run --release --bin mina-node-testing -- scenarios-run --name scenario-name +//! ``` + mod exit_with_error; use std::sync::Arc; diff --git a/node/testing/src/main.rs b/node/testing/src/main.rs index 02917a3412..8068bc25f1 100644 --- a/node/testing/src/main.rs +++ b/node/testing/src/main.rs @@ -1,3 +1,27 @@ +//! # Mina Node Testing CLI +//! +//! Command-line interface for running Mina node scenario tests. +//! Provides tools for generating, running, and managing deterministic +//! blockchain testing scenarios. +//! +//! ## Documentation +//! +//! For detailed documentation and usage examples, see: +//! - [Scenario Tests](https://o1-labs.github.io/mina-rust/developers/testing/scenario-tests) - Complete testing guide +//! - [Testing Framework](https://o1-labs.github.io/mina-rust/developers/testing/testing-framework) - Testing architecture +//! +//! ## Quick Start +//! +//! List all available scenarios: +//! ```bash +//! cargo run --release --bin mina-node-testing -- scenarios-list +//! ``` +//! +//! Run a specific scenario: +//! ```bash +//! cargo run --release --bin mina-node-testing -- scenarios-run --name p2p-signaling +//! ``` + use clap::Parser; use mina_node_testing::{ diff --git a/node/testing/src/scenarios/mod.rs b/node/testing/src/scenarios/mod.rs index 8b2ce48a60..095f36b770 100644 --- a/node/testing/src/scenarios/mod.rs +++ b/node/testing/src/scenarios/mod.rs @@ -1,13 +1,45 @@ -//! Basic connectivity tests. -//! Initial Joining: +//! # Mina Testing Scenarios +//! +//! This module contains scenario-based tests for the Mina Rust node implementation. +//! Scenarios are deterministic, ordered sequences of steps that test complex +//! multi-node blockchain interactions. +//! +//! ## Documentation +//! +//! For comprehensive documentation on the testing framework and available +//! scenarios, see: +//! - **[Scenario Tests](https://o1-labs.github.io/mina-rust/developers/testing/scenario-tests)** - Main scenario testing documentation +//! - **[Testing Framework](https://o1-labs.github.io/mina-rust/developers/testing/testing-framework)** - Overall testing architecture +//! - **[Network Connectivity](https://o1-labs.github.io/mina-rust/developers/testing/network-connectivity)** - Network connectivity testing details +//! +//! ## Usage +//! +//! List available scenarios: +//! ```bash +//! cargo run --release --bin mina-node-testing -- scenarios-list +//! ``` +//! +//! Run a specific scenario: +//! ```bash +//! cargo run --release --bin mina-node-testing -- scenarios-run --name scenario-name +//! ``` +//! +//! ## Test Categories +//! +//! Basic connectivity tests: //! * Ensure new nodes can discover peers and establish initial connections. -//! * Test how nodes handle scenarios when they are overwhelmed with too many connections or data requests. +//! * Test how nodes handle scenarios when they are overwhelmed with too many +//! connections or data requests. //! //! TODO(vlad9486): -//! Reconnection: Validate that nodes can reconnect after both intentional and unintentional disconnections. -//! Handling Latency: Nodes should remain connected and synchronize even under high latency conditions. -//! Intermittent Connections: Nodes should be resilient to sporadic network dropouts and still maintain synchronization. -//! Dynamic IP Handling: Nodes with frequently changing IP addresses should maintain stable connections. +//! - Reconnection: Validate that nodes can reconnect after both intentional and +//! unintentional disconnections. +//! - Handling Latency: Nodes should remain connected and synchronize even under +//! high latency conditions. +//! - Intermittent Connections: Nodes should be resilient to sporadic network +//! dropouts and still maintain synchronization. +//! - Dynamic IP Handling: Nodes with frequently changing IP addresses should +//! maintain stable connections. pub mod multi_node; pub mod record_replay; From a1e72e901624eac39ee9f3d64bb6838f0539a695 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 21:28:22 +0200 Subject: [PATCH 06/17] Website/scenario-tests: add differences between CI and local exec --- .../docs/developers/testing/scenario-tests.md | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/website/docs/developers/testing/scenario-tests.md b/website/docs/developers/testing/scenario-tests.md index d01f5a1d81..728e7f85b8 100644 --- a/website/docs/developers/testing/scenario-tests.md +++ b/website/docs/developers/testing/scenario-tests.md @@ -321,6 +321,75 @@ cargo run --release --features scenario-generators,p2p-webrtc \ --bin mina-node-testing -- scenarios-list ``` +## CI vs Local Test Execution + +There are important differences between how tests run in CI versus locally: + +### CI Environment (with Sidecar Container) + +In CI, tests run with a +**[network debugger sidecar container](https://github.com/openmina/mina-network-debugger)** +that provides deep network inspection capabilities: + +- **Network monitoring**: A `bpf-recorder` sidecar container runs alongside test + nodes +- **Binary execution**: CI builds and uses specific scenario binaries based on + directory structure: + - Built using `make build-tests` and `make build-tests-webrtc` + - `solo_node/` directory scenarios → dedicated solo node test binaries + - `multi_node/` directory scenarios → multi-node test binaries + - `p2p/` directory scenarios → P2P networking test binaries +- **Debugger integration**: Automatic network traffic capture and analysis +- **Port configuration**: Debugger runs on port 8000 (configured for CI + environment) +- **Enhanced observability**: Full message tracing, connection monitoring, and + protocol analysis + +The sidecar container provides: + +- Real-time network connection tracking +- Message-level protocol inspection +- Connection lifecycle monitoring +- Stream-level debugging capabilities +- HTTP API for accessing captured network data + +### Local Environment (Scenario-run) + +Locally, tests run through the scenario-run interface with simplified execution: + +- **Scenario names**: Tests are referenced by name (e.g., `p2p-signaling`, + `solo-node-bootstrap`) +- **Direct execution**: Single binary (`mina-node-testing`) handles all scenario + types +- **Optional debugger**: Network debugger can be enabled with `--use-debugger` + flag +- **Simplified setup**: No external container dependencies +- **Development focus**: Optimized for rapid iteration and debugging + +### Key Differences + +| Aspect | CI (Sidecar) | Local (Scenario-run) | +| ----------------- | ------------------------ | --------------------------- | +| **Execution** | Directory-based binaries | Name-based scenarios | +| **Network Debug** | Always enabled (sidecar) | Optional (`--use-debugger`) | +| **Observability** | Full network inspection | Basic logging | +| **Setup** | Container orchestration | Single binary | +| **Use Case** | Comprehensive testing | Development iteration | + +### Running with Network Debugger Locally + +To enable similar debugging capabilities locally: + +```bash +# Enable network debugger for detailed inspection +cargo run --release --bin mina-node-testing -- \ + scenarios-generate --use-debugger --name scenario-name +``` + +The local debugger spawns a `bpf-recorder` process that provides similar network +monitoring capabilities as the CI sidecar, though without the container +isolation. + ### Scenario generation and replay The `mina-node-testing` framework supports both scenario generation and replay: From 7c38d7fd8ead8ff060866a7a8f47313ca0fd906f Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 21:28:42 +0200 Subject: [PATCH 07/17] Node/testing: drone_ci is an outdated name, add TODO --- node/testing/src/network_debugger.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/node/testing/src/network_debugger.rs b/node/testing/src/network_debugger.rs index e2cf1bbed1..b2bd99ae71 100644 --- a/node/testing/src/network_debugger.rs +++ b/node/testing/src/network_debugger.rs @@ -53,6 +53,10 @@ pub enum StreamId { } impl Debugger { + // TODO: Rename `drone_ci` to `external_ci` or `ci_sidecar` since the project + // no longer uses Drone CI. This method connects to an external debugger + // service (like the mina-network-debugger sidecar container) rather than + // spawning a local debugger process. pub fn drone_ci() -> Self { Debugger { child: None, From 9d3fcb26e31151071af74af7e2b2ee622f0bc155 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 21:29:09 +0200 Subject: [PATCH 08/17] Node/scenario: top-level doc reg. CI vs local exec --- node/testing/src/scenarios/mod.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/node/testing/src/scenarios/mod.rs b/node/testing/src/scenarios/mod.rs index 095f36b770..83ba7f4cee 100644 --- a/node/testing/src/scenarios/mod.rs +++ b/node/testing/src/scenarios/mod.rs @@ -26,7 +26,22 @@ //! //! ## Test Categories //! -//! Basic connectivity tests: +//! ### Directory Structure and CI Integration +//! +//! The directory structure in this module maps to specific test execution contexts: +//! +//! - **`solo_node/`** - Single node scenarios, often testing OCaml interoperability +//! - **`multi_node/`** - Multi-node Rust network scenarios +//! - **`p2p/`** - Low-level P2P networking and protocol tests +//! - **`record_replay/`** - Scenario recording and replay functionality +//! - **`simulation/`** - Large-scale network simulations +//! +//! In CI environments, these directories determine which specialized test binaries +//! are used for execution, while locally all scenarios are accessible through the +//! unified `mina-node-testing` CLI using scenario names. +//! +//! ### Basic Connectivity Tests +//! //! * Ensure new nodes can discover peers and establish initial connections. //! * Test how nodes handle scenarios when they are overwhelmed with too many //! connections or data requests. From 2ee97206ee4903f8208f219c7a6ceff23af04848 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 22:08:18 +0200 Subject: [PATCH 09/17] Node/testing: add an option to output scenarii to JSON Default option is the previous behavior, which is stdout --- node/testing/src/main.rs | 31 +++++++++++++-- .../docs/developers/testing/scenario-tests.md | 39 +++++++++++++++---- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/node/testing/src/main.rs b/node/testing/src/main.rs index 8068bc25f1..c56e20dbb8 100644 --- a/node/testing/src/main.rs +++ b/node/testing/src/main.rs @@ -70,6 +70,14 @@ pub struct CommandScenariosGenerate { pub use_debugger: bool, #[arg(long, short)] pub webrtc: bool, + #[arg(long, short = 'o', default_value = "stdout", value_enum)] + pub output: OutputFormat, +} + +#[derive(Debug, Clone, clap::ValueEnum)] +pub enum OutputFormat { + Stdout, + Json, } /// Run scenario located at `res/scenarios`. @@ -111,6 +119,7 @@ impl Command { Self::ScenariosGenerate(cmd) => { #[cfg(feature = "scenario-generators")] { + let output_format = cmd.output.clone(); let run_scenario = |scenario: Scenarios| -> Result<_, anyhow::Error> { let mut config = scenario.default_cluster_config()?; if cmd.use_debugger { @@ -119,18 +128,34 @@ impl Command { if cmd.webrtc { config.set_all_rust_to_rust_use_webrtc(); } - Ok(scenario.run_only_from_scratch(config)) + Ok((scenario, config)) }; let fut = async move { if let Some(name) = cmd.name { if let Some(scenario) = Scenarios::find_by_name(&name) { - run_scenario(scenario)?.await; + let (scenario, config) = run_scenario(scenario)?; + match output_format { + OutputFormat::Json => { + scenario.run_and_save_from_scratch(config).await + } + OutputFormat::Stdout => { + scenario.run_only_from_scratch(config).await + } + } } else { anyhow::bail!("no such scenario: \"{name}\""); } } else { for scenario in Scenarios::iter() { - run_scenario(scenario)?.await; + let (scenario, config) = run_scenario(scenario)?; + match output_format { + OutputFormat::Json => { + scenario.run_and_save_from_scratch(config).await + } + OutputFormat::Stdout => { + scenario.run_only_from_scratch(config).await + } + } } } Ok(()) diff --git a/website/docs/developers/testing/scenario-tests.md b/website/docs/developers/testing/scenario-tests.md index 728e7f85b8..c064d14b8e 100644 --- a/website/docs/developers/testing/scenario-tests.md +++ b/website/docs/developers/testing/scenario-tests.md @@ -390,24 +390,46 @@ The local debugger spawns a `bpf-recorder` process that provides similar network monitoring capabilities as the CI sidecar, though without the container isolation. +## Troubleshooting + +### Workflow Requirements + +- **scenarios-run**: Expects pre-existing scenario files in + `node/testing/res/scenarios/` +- **scenarios-generate**: + - Default (`--output=stdout`): Runs scenarios and outputs to stdout, no JSON + files created + - With `--output=json`: Runs scenarios and saves them as JSON files in + `node/testing/res/scenarios/` + +#### Understanding Scenario Load/Save Implementation + +For detailed technical information about how scenarios are loaded and saved, see +the +[scenario module source code](https://github.com/o1-labs/mina-rust/blob/develop/node/testing/src/scenario/mod.rs). + ### Scenario generation and replay The `mina-node-testing` framework supports both scenario generation and replay: ```bash -# Generate specific scenarios (requires scenario-generators feature) +# Generate and run scenarios (default: output to stdout, no JSON file saved) cargo run --release --features scenario-generators --bin mina-node-testing -- \ scenarios-generate --name record-replay-block-production +# Generate and save scenarios to JSON files +cargo run --release --features scenario-generators --bin mina-node-testing -- \ + scenarios-generate --name record-replay-block-production --output=json + # Generate WebRTC scenarios (requires additional p2p-webrtc feature) cargo run --release --features scenario-generators,p2p-webrtc \ - --bin mina-node-testing -- scenarios-generate --name p2p-signaling + --bin mina-node-testing -- scenarios-generate --name p2p-signaling --output=json -# Generate multi-node scenarios +# Generate all scenarios and save to JSON cargo run --release --features scenario-generators --bin mina-node-testing -- \ - scenarios-generate --name multi-node-pubsub-propagate-block + scenarios-generate --output=json -# Replay existing scenarios (use actual scenario names from list) +# Replay existing scenarios (requires JSON files from scenarios-generate --output=json) cargo run --release --bin mina-node-testing -- scenarios-run --name p2p-signaling ``` @@ -415,8 +437,11 @@ cargo run --release --bin mina-node-testing -- scenarios-run --name p2p-signalin - **Feature-based compilation**: Use `--features` to enable specific test capabilities -- **Scenario generation**: Create new test scenarios from templates -- **Scenario replay**: Execute previously generated or recorded scenarios +- **Scenario generation**: + - Run scenarios directly with `--output=stdout` (default) + - Generate JSON files with `--output=json` for later replay +- **Scenario replay**: Execute previously generated JSON scenarios using + `scenarios-run` - **Named scenarios**: Reference scenarios by name for consistent execution ### Environment configuration From 5d276eb8313c4539480426f606068d1de25f3cfe Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 22:08:59 +0200 Subject: [PATCH 10/17] Node/testing: better error handlings and documentation --- node/testing/src/scenario/mod.rs | 163 +++++++++++++++++++++++++++--- node/testing/src/scenarios/mod.rs | 10 +- 2 files changed, 155 insertions(+), 18 deletions(-) diff --git a/node/testing/src/scenario/mod.rs b/node/testing/src/scenario/mod.rs index 52f321ede6..5a76e2fdde 100644 --- a/node/testing/src/scenario/mod.rs +++ b/node/testing/src/scenario/mod.rs @@ -1,3 +1,33 @@ +//! # Scenario Management +//! +//! This module provides functionality for managing test scenarios, including +//! loading, saving, and executing deterministic test sequences. +//! +//! ## How Scenarios Work +//! +//! ### Storage Format +//! Scenarios are stored as JSON files in the `res/scenarios/` directory relative +//! to the testing crate. Each scenario file contains: +//! - **ScenarioInfo**: Metadata (ID, description, parent relationships, node configs) +//! - **ScenarioSteps**: Ordered sequence of test actions to execute +//! +//! ### Load Process +//! 1. **File Location**: `load()` reads from `{CARGO_MANIFEST_DIR}/res/scenarios/{id}.json` +//! 2. **JSON Parsing**: Deserializes the file into a `Scenario` struct +//! 3. **Error Handling**: Returns `anyhow::Error` if file doesn't exist or is malformed +//! +//! ### Save Process +//! 1. **Atomic Write**: Uses temporary file + rename for atomic operations +//! 2. **Directory Creation**: Automatically creates `res/scenarios/` if needed +//! 3. **JSON Format**: Pretty-prints JSON for human readability +//! 4. **Temporary Files**: `.tmp.{scenario_id}.json` during write, renamed on success +//! +//! ### Scenario Inheritance +//! Scenarios can have parent-child relationships where child scenarios inherit +//! setup steps from their parents, enabling composition and reuse. +//! +//! For usage examples, see the [testing documentation](https://o1-labs.github.io/mina-rust/developers/testing/scenario-tests). + mod id; pub use id::ScenarioId; @@ -7,6 +37,8 @@ pub use step::{ListenerNode, ScenarioStep}; mod event_details; pub use event_details::event_details; +use anyhow::Context; +use mina_core::log::{debug, info, system_time}; use serde::{Deserialize, Serialize}; use crate::node::NodeTestingConfig; @@ -71,42 +103,147 @@ impl Scenario { } pub async fn list() -> Result, anyhow::Error> { - let mut files = tokio::fs::read_dir(Self::PATH).await?; + let mut files = tokio::fs::read_dir(Self::PATH).await.with_context(|| { + format!( + "Failed to read scenarios directory '{}'. Ensure the directory \ + exists or create it with: mkdir -p {}", + Self::PATH, + Self::PATH + ) + })?; let mut list = vec![]; while let Some(file) = files.next_entry().await? { - let encoded = tokio::fs::read(file.path()).await?; + let file_path = file.path(); + let encoded = tokio::fs::read(&file_path).await.with_context(|| { + format!("Failed to read scenario file '{}'", file_path.display()) + })?; // TODO(binier): maybe somehow only parse info part of json? - let full: Self = serde_json::from_slice(&encoded)?; + let full: Self = serde_json::from_slice(&encoded).with_context(|| { + format!( + "Failed to parse scenario file '{}' as valid JSON", + file_path.display() + ) + })?; list.push(full.info); } Ok(list) } + /// Load a scenario from disk by ID. + /// + /// This method reads the scenario file from `res/scenarios/{id}.json`, + /// deserializes it from JSON, and returns the complete scenario including + /// both metadata and steps. + /// + /// # Arguments + /// * `id` - The scenario identifier used to construct the file path + /// + /// # Returns + /// * `Ok(Scenario)` - Successfully loaded scenario + /// * `Err(anyhow::Error)` - File not found, invalid JSON, or I/O error pub async fn load(id: &ScenarioId) -> Result { - let encoded = tokio::fs::read(Self::file_path_by_id(id)).await?; - Ok(serde_json::from_slice(&encoded)?) + let path = Self::file_path_by_id(id); + debug!(system_time(); "Loading scenario '{}' from file '{}'", id, path); + let encoded = tokio::fs::read(&path).await.with_context(|| { + format!( + "Failed to read scenario file '{}'. Ensure the scenario exists. \ + If using scenarios-run, the scenario must be generated first using \ + scenarios-generate, or check if the required feature flags (like \ + 'p2p-webrtc') are enabled", + path + ) + })?; + let scenario = serde_json::from_slice(&encoded) + .with_context(|| format!("Failed to parse scenario file '{}' as valid JSON", path))?; + info!(system_time(); "Successfully loaded scenario '{}'", id); + Ok(scenario) } + /// Reload this scenario from disk, discarding any in-memory changes. pub async fn reload(&mut self) -> Result<(), anyhow::Error> { *self = Self::load(&self.info.id).await?; Ok(()) } + /// Save the scenario to disk using atomic write operations. + /// + /// This method implements atomic writes by: + /// 1. Creating the scenarios directory if it doesn't exist + /// 2. Writing to a temporary file (`.tmp.{id}.json`) + /// 3. Pretty-printing JSON for human readability + /// 4. Atomically renaming the temp file to the final name + /// + /// This ensures the scenario file is never in a partially-written state, + /// preventing corruption during concurrent access or system crashes. + /// + /// # File Location + /// Saves to: `{CARGO_MANIFEST_DIR}/res/scenarios/{id}.json` + /// + /// # Errors + /// Returns error if: + /// - Cannot create the scenarios directory + /// - Cannot serialize scenario to JSON + /// - File I/O operations fail + /// - Atomic rename fails pub async fn save(&self) -> Result<(), anyhow::Error> { let tmp_file = self.tmp_file_path(); - let encoded = serde_json::to_vec_pretty(self)?; - tokio::fs::create_dir_all(Self::PATH).await?; - tokio::fs::write(&tmp_file, encoded).await?; - Ok(tokio::fs::rename(tmp_file, self.file_path()).await?) + let final_file = self.file_path(); + + debug!(system_time(); "Saving scenario '{}' to file '{}'", self.info.id, final_file); + + let encoded = serde_json::to_vec_pretty(self) + .with_context(|| format!("Failed to serialize scenario '{}' to JSON", self.info.id))?; + + tokio::fs::create_dir_all(Self::PATH) + .await + .with_context(|| format!("Failed to create scenarios directory '{}'", Self::PATH))?; + + tokio::fs::write(&tmp_file, encoded) + .await + .with_context(|| format!("Failed to write temporary scenario file '{}'", tmp_file))?; + + tokio::fs::rename(&tmp_file, &final_file) + .await + .with_context(|| { + format!( + "Failed to rename temporary file '{}' to final scenario file '{}'", + tmp_file, final_file + ) + })?; + + info!(system_time(); "Successfully saved scenario '{}'", self.info.id); + Ok(()) } + /// Synchronous version of `save()` for use in non-async contexts. + /// + /// Implements the same atomic write pattern as `save()` but uses + /// blocking I/O operations instead of async. pub fn save_sync(&self) -> Result<(), anyhow::Error> { let tmp_file = self.tmp_file_path(); - let encoded = serde_json::to_vec_pretty(self)?; - std::fs::create_dir_all(Self::PATH)?; - std::fs::write(&tmp_file, encoded)?; - Ok(std::fs::rename(tmp_file, self.file_path())?) + let final_file = self.file_path(); + + debug!(system_time(); "Saving scenario '{}' to file '{}'", self.info.id, final_file); + + let encoded = serde_json::to_vec_pretty(self) + .with_context(|| format!("Failed to serialize scenario '{}' to JSON", self.info.id))?; + + std::fs::create_dir_all(Self::PATH) + .with_context(|| format!("Failed to create scenarios directory '{}'", Self::PATH))?; + + std::fs::write(&tmp_file, encoded) + .with_context(|| format!("Failed to write temporary scenario file '{}'", tmp_file))?; + + std::fs::rename(&tmp_file, &final_file).with_context(|| { + format!( + "Failed to rename temporary file '{}' to final scenario file '{}'", + tmp_file, final_file + ) + })?; + + info!(system_time(); "Successfully saved scenario '{}'", self.info.id); + Ok(()) } } diff --git a/node/testing/src/scenarios/mod.rs b/node/testing/src/scenarios/mod.rs index 83ba7f4cee..cb27ecb87e 100644 --- a/node/testing/src/scenarios/mod.rs +++ b/node/testing/src/scenarios/mod.rs @@ -44,17 +44,17 @@ //! //! * Ensure new nodes can discover peers and establish initial connections. //! * Test how nodes handle scenarios when they are overwhelmed with too many -//! connections or data requests. +//! connections or data requests. //! //! TODO(vlad9486): //! - Reconnection: Validate that nodes can reconnect after both intentional and -//! unintentional disconnections. +//! unintentional disconnections. //! - Handling Latency: Nodes should remain connected and synchronize even under -//! high latency conditions. +//! high latency conditions. //! - Intermittent Connections: Nodes should be resilient to sporadic network -//! dropouts and still maintain synchronization. +//! dropouts and still maintain synchronization. //! - Dynamic IP Handling: Nodes with frequently changing IP addresses should -//! maintain stable connections. +//! maintain stable connections. pub mod multi_node; pub mod record_replay; From 439121a7701a803310530a8c8ef3a1c23179d818 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 22:26:39 +0200 Subject: [PATCH 11/17] Node/testing: using logger instead of stdout with eprintln --- node/testing/src/scenarios/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/node/testing/src/scenarios/mod.rs b/node/testing/src/scenarios/mod.rs index cb27ecb87e..3d1561e2fe 100644 --- a/node/testing/src/scenarios/mod.rs +++ b/node/testing/src/scenarios/mod.rs @@ -68,6 +68,7 @@ pub use driver::*; pub use crate::cluster::runner::*; +use mina_core::log::{debug, system_time, warn}; use strum_macros::{EnumIter, EnumString, IntoStaticStr}; use crate::{ @@ -354,9 +355,9 @@ impl Scenarios { let steps = std::mem::take(&mut self.0.steps); let scenario = Scenario { info, steps }; - eprintln!("saving scenario({}) before exit...", scenario.info.id); + debug!(system_time(); "saving scenario({}) before exit...", scenario.info.id); if let Err(err) = scenario.save_sync() { - eprintln!( + warn!(system_time(); "failed to save scenario({})! error: {}", scenario.info.id, err ); @@ -364,7 +365,7 @@ impl Scenarios { } } - eprintln!("run_and_save: {}", self.to_str()); + debug!(system_time(); "run_and_save: {}", self.to_str()); let mut scenario = ScenarioSaveOnExit(self.blank_scenario()); self.run(cluster, |step| scenario.0.add_step(step.clone()).unwrap()) .await; @@ -373,7 +374,7 @@ impl Scenarios { } pub async fn run_only(self, cluster: &mut Cluster) { - eprintln!("run_only: {}", self.to_str()); + debug!(system_time(); "run_only: {}", self.to_str()); self.run(cluster, |_| {}).await } From 2c8bc80496e47d04206db869b81eb6a5826abb27 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 22:31:02 +0200 Subject: [PATCH 12/17] Node/testing: Openmina -> Rust node --- .../testing/src/scenarios/solo_node/sync_root_snarked_ledger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/testing/src/scenarios/solo_node/sync_root_snarked_ledger.rs b/node/testing/src/scenarios/solo_node/sync_root_snarked_ledger.rs index 6b718972e5..972505f67d 100644 --- a/node/testing/src/scenarios/solo_node/sync_root_snarked_ledger.rs +++ b/node/testing/src/scenarios/solo_node/sync_root_snarked_ledger.rs @@ -34,7 +34,7 @@ pub struct SoloNodeSyncRootSnarkedLedger; impl SoloNodeSyncRootSnarkedLedger { pub async fn run(self, mut runner: ClusterRunner<'_>) { let node_id = runner.add_rust_node(RustNodeTestingConfig::devnet_default()); - eprintln!("launch Openmina node with default configuration, id: {node_id}"); + eprintln!("launch Rust node with default configuration, id: {node_id}"); const REPLAYER_1: &str = "/dns4/web-node-1/tcp/18302/p2p/12D3KooWD8jSyPFXNdAcMBHyHjRBcK1AW9t3xvnpfCFSRKMweVKi"; From bc07def6a57f761785f9bbc8c99b4a7ae7f93416 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 23:02:04 +0200 Subject: [PATCH 13/17] Node/testing: remove old commented default docker image --- node/testing/src/node/ocaml/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/node/testing/src/node/ocaml/config.rs b/node/testing/src/node/ocaml/config.rs index a242cd4a40..e1342cba64 100644 --- a/node/testing/src/node/ocaml/config.rs +++ b/node/testing/src/node/ocaml/config.rs @@ -129,7 +129,6 @@ impl OcamlNodeConfig { } impl OcamlNodeExecutable { - // pub const DEFAULT_DOCKER_IMAGE: &'static str = "vladsimplestakingcom/mina-light:2.0.0rampup4"; pub const DEFAULT_DOCKER_IMAGE: &'static str = "gcr.io/o1labs-192920/mina-daemon:3.2.0-beta2-939b08d-noble-devnet"; pub const DEFAULT_MINA_EXECUTABLE: &'static str = "mina"; From 8357eaf3a26f82282b8f799811e24e0f8c00861e Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 23:16:47 +0200 Subject: [PATCH 14/17] Node/testing: adding logging when spawning OCaml process --- node/testing/src/node/ocaml/mod.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/node/testing/src/node/ocaml/mod.rs b/node/testing/src/node/ocaml/mod.rs index cb547bc339..e0d277a505 100644 --- a/node/testing/src/node/ocaml/mod.rs +++ b/node/testing/src/node/ocaml/mod.rs @@ -2,9 +2,14 @@ mod config; pub use config::*; use mina_core::{thread, ChainId}; use mina_p2p_messages::v2::StateHash; -use node::p2p::{ - connection::outgoing::{P2pConnectionOutgoingInitLibp2pOpts, P2pConnectionOutgoingInitOpts}, - PeerId, +use node::{ + core::log::{info, system_time}, + p2p::{ + connection::outgoing::{ + P2pConnectionOutgoingInitLibp2pOpts, P2pConnectionOutgoingInitOpts, + }, + PeerId, + }, }; use std::{ @@ -126,7 +131,9 @@ impl OcamlNode { cmd.stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); + info!(system_time(); "Spawning OCaml daemon process"); let mut child = cmd.spawn()?; + info!(system_time(); "OCaml daemon process started with PID: {:?}", child.id()); let stdout = child .stdout From a18d83b47b032b172a2b6060215baf428dff07d1 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 23:17:28 +0200 Subject: [PATCH 15/17] Node/testing/cluster: add comprehensive doc + logging --- node/testing/src/cluster/mod.rs | 277 ++++++++++++++++++++++++++++++-- 1 file changed, 264 insertions(+), 13 deletions(-) diff --git a/node/testing/src/cluster/mod.rs b/node/testing/src/cluster/mod.rs index c79745e3b0..bc5f838a8a 100644 --- a/node/testing/src/cluster/mod.rs +++ b/node/testing/src/cluster/mod.rs @@ -1,3 +1,35 @@ +//! Cluster Management for Multi-Node Testing +//! +//! This module provides the core infrastructure for managing clusters of +//! Mina nodes during testing scenarios. It supports both Rust and OCaml +//! node implementations, enabling cross-implementation testing and complex +//! multi-node scenarios. +//! +//! # Key Components +//! +//! - [`Cluster`] - Main cluster coordinator managing node lifecycle +//! - Node addition methods for different node types +//! - Port allocation and resource management +//! - Scenario execution and state tracking +//! - Network debugger integration +//! +//! # Node Addition Methods +//! +//! - [`Cluster::add_rust_node`] - Add Rust implementation nodes +//! - [`Cluster::add_ocaml_node`] - Add OCaml implementation nodes +//! +//! # Example +//! +//! ```rust,no_run +//! let mut cluster = Cluster::new(ClusterConfig::default()); +//! +//! // Add Rust node with custom configuration +//! let rust_node = cluster.add_rust_node(RustNodeTestingConfig::default()); +//! +//! // Add OCaml node for cross-implementation testing +//! let ocaml_node = cluster.add_ocaml_node(OcamlNodeTestingConfig::default()); +//! ``` + mod config; pub use config::{ClusterConfig, ProofKind}; @@ -25,8 +57,12 @@ use mina_node_native::{http_server, NodeServiceBuilder}; use node::{ account::{AccountPublicKey, AccountSecretKey}, core::{ - consensus::ConsensusConstants, constants::constraint_constants, - invariants::InvariantsState, log::system_time, requests::RpcId, thread, warn, + consensus::ConsensusConstants, + constants::constraint_constants, + invariants::InvariantsState, + log::{info, system_time, warn}, + requests::RpcId, + thread, }, event_source::Event, p2p::{ @@ -117,30 +153,72 @@ lazy_static::lazy_static! { static ref VERIFIER_SRS: Arc = get_srs(); } +/// Manages a cluster of Mina nodes for testing scenarios. +/// +/// The `Cluster` struct coordinates multiple node instances, handling +/// resource allocation, configuration, and lifecycle management. It supports +/// both Rust and OCaml node implementations for comprehensive testing. +/// +/// # Default Behaviors +/// +/// - **Port allocation**: Automatically assigns available ports from the +/// configured range, testing availability before assignment +/// - **Keypair management**: Uses deterministic keypairs for Rust nodes and +/// rotates through predefined keypairs for OCaml nodes +/// - **Resource isolation**: Each node gets isolated temporary directories +/// - **Verifier indices**: Shared verifier SRS and indices across all nodes +/// - **Network debugging**: Optional debugger integration for CI environments +/// +/// # Node Addition +/// +/// The cluster provides specialized methods for adding different node types: +/// - Rust nodes via [`add_rust_node`](Self::add_rust_node) +/// - OCaml nodes via [`add_ocaml_node`](Self::add_ocaml_node) pub struct Cluster { + /// Cluster-wide configuration settings pub config: ClusterConfig, + /// Current scenario execution state scenario: ClusterScenarioRun, + /// Iterator over available ports for node allocation available_ports: Box + Send>, + /// Registry of account secret keys for deterministic testing account_sec_keys: BTreeMap, + /// Collection of active Rust nodes nodes: Vec, + /// Collection of active OCaml nodes (Option for lifecycle management) ocaml_nodes: Vec>, + /// Genesis timestamp for deterministic time progression initial_time: Option, + /// Counter for generating unique RPC request IDs rpc_counter: usize, + /// Index for rotating OCaml LibP2P keypairs ocaml_libp2p_keypair_i: usize, + /// Shared verifier SRS for proof verification verifier_srs: Arc, + /// Block verifier index for consensus validation block_verifier_index: BlockVerifier, + /// Transaction verifier index for transaction validation work_verifier_index: TransactionVerifier, + /// Optional network traffic debugger debugger: Option, + /// Shared state for invariant checking across nodes invariants_state: Arc>, } +/// Tracks the execution state of scenario chains within a cluster. +/// +/// Manages the progression through scenario steps and maintains history +/// of completed scenarios for debugging and analysis. #[derive(Serialize)] pub struct ClusterScenarioRun { + /// Queue of scenarios to be executed (supports scenario inheritance) chain: VecDeque, + /// History of completed scenarios finished: Vec, + /// Current step index within the active scenario cur_step: usize, } @@ -201,16 +279,70 @@ impl Cluster { self.initial_time } + /// Add a new Rust implementation node to the cluster. + /// + /// Creates and configures a Rust Mina node with the specified testing + /// configuration. This method handles all aspects of node initialization + /// including port allocation, key generation, service setup, and state + /// initialization. + /// + /// # Default Behaviors + /// + /// - **Port allocation**: HTTP and LibP2P ports automatically assigned + /// from available port range + /// - **Peer identity**: Deterministic LibP2P keypair based on node index + /// - **Work directory**: Isolated temporary directory per node + /// - **Invariants**: Automatic invariant checking enabled + /// - **HTTP server**: Spawned on separate thread for API access + /// - **Proof verification**: Shared verifier indices and SRS + /// + /// # Configuration Options + /// + /// - `peer_id`: Deterministic or custom LibP2P identity + /// - `libp2p_port`: Custom P2P port (auto-assigned if None) + /// - `initial_peers`: Peer connection targets (supports node references) + /// - `block_producer`: Optional block production configuration + /// - `genesis`: Genesis ledger and protocol constants + /// - `snark_worker`: SNARK work generation settings + /// + /// # Returns + /// + /// Returns a [`ClusterNodeId`] that can be used to reference this node + /// in scenarios and for inter-node connections. + /// + /// # Panics + /// + /// Panics if: + /// - No available ports in the configured range + /// - Node service initialization fails + /// - Invalid genesis configuration pub fn add_rust_node(&mut self, testing_config: RustNodeTestingConfig) -> ClusterNodeId { let rng_seed = [0; 32]; let node_config = testing_config.clone(); let node_id = ClusterNodeId::new_unchecked(self.nodes.len()); + + info!( + system_time(); + "Adding Rust node {} with config: max_peers={}, snark_worker={:?}, \ + block_producer={}", + node_id.index(), + testing_config.max_peers, + testing_config.snark_worker, + testing_config.block_producer.is_some() + ); + let work_dir = TempDir::new().unwrap(); let shutdown_initiator = Aborter::default(); let shutdown_listener = shutdown_initiator.aborted(); let p2p_sec_key = match testing_config.peer_id { - TestPeerId::Derived => P2pSecretKey::deterministic(node_id.index()), - TestPeerId::Bytes(bytes) => P2pSecretKey::from_bytes(bytes), + TestPeerId::Derived => { + info!(system_time(); "Using deterministic peer ID for node {}", node_id.index()); + P2pSecretKey::deterministic(node_id.index()) + } + TestPeerId::Bytes(bytes) => { + info!(system_time(); "Using custom peer ID for node {}", node_id.index()); + P2pSecretKey::from_bytes(bytes) + } }; let http_port = self @@ -235,21 +367,60 @@ impl Cluster { .unwrap() }); + info!( + system_time(); + "Assigned ports for Rust node {}: HTTP={}, LibP2P={}", + node_id.index(), + http_port, + libp2p_port + ); + let (block_producer_sec_key, block_producer_config) = testing_config .block_producer - .map(|v| (v.sec_key, v.config)) + .map(|v| { + info!( + system_time(); + "Configuring block producer for Rust node {} with public key: {}", + node_id.index(), + v.sec_key.public_key() + ); + (v.sec_key, v.config) + }) .unzip(); - let initial_peers = testing_config + let initial_peers: Vec<_> = testing_config .initial_peers .into_iter() - .map(|node| match node { - ListenerNode::Rust(id) => self.node(id).unwrap().dial_addr(), - ListenerNode::Ocaml(id) => self.ocaml_node(id).unwrap().dial_addr(), - ListenerNode::Custom(addr) => addr, + .map(|node| { + let addr = match &node { + ListenerNode::Rust(id) => { + info!(system_time(); "Adding Rust peer {} as initial peer", id.index()); + self.node(*id).unwrap().dial_addr() + } + ListenerNode::Ocaml(id) => { + info!(system_time(); "Adding OCaml peer {} as initial peer", id.index()); + self.ocaml_node(*id).unwrap().dial_addr() + } + ListenerNode::Custom(addr) => { + info!(system_time(); "Adding custom peer: {:?}", addr); + addr.clone() + } + }; + addr }) .collect(); + if !initial_peers.is_empty() { + info!( + system_time(); + "Rust node {} configured with {} initial peers", + node_id.index(), + initial_peers.len() + ); + } else { + info!(system_time(); "Rust node {} configured as seed node (no initial peers)", node_id.index()); + } + let protocol_constants = testing_config .genesis .protocol_constants() @@ -317,6 +488,7 @@ impl Cluster { }); if let Some(keypair) = block_producer_sec_key { + info!(system_time(); "Initializing block producer for Rust node {}", node_id.index()); let provers = BlockProver::make(None, None); service_builder.block_producer_init(keypair, Some(provers)); } @@ -405,13 +577,72 @@ impl Cluster { let node = Node::new(work_dir, node_config, store); + info!( + system_time(); + "Successfully created Rust node {} at ports HTTP={}, LibP2P={}", + node_id.index(), + http_port, + libp2p_port + ); + self.nodes.push(node); node_id } + /// Add a new OCaml implementation node to the cluster. + /// + /// Creates and spawns an OCaml Mina daemon process with the specified + /// configuration. This method handles process spawning, port allocation, + /// directory setup, and daemon configuration. + /// + /// # Default Behaviors + /// + /// - **Executable selection**: Automatically detects local binary or + /// falls back to default Docker image + /// - **Port allocation**: LibP2P, GraphQL, and client ports automatically + /// assigned from available range + /// - **Keypair rotation**: Uses predefined LibP2P keypairs, rotating + /// through the set for each new node + /// - **Process management**: Spawns daemon with proper environment + /// variables and argument configuration + /// - **Logging**: Stdout/stderr forwarded with port-based prefixes + /// - **Docker support**: Automatic container management when using Docker + /// + /// # Configuration Options + /// + /// - `initial_peers`: List of peer connection targets + /// - `daemon_json`: Genesis configuration (file path or in-memory JSON) + /// - `block_producer`: Optional block production key + /// + /// # Docker vs Local Execution + /// + /// The method automatically determines execution mode: + /// 1. Attempts to use locally installed `mina` binary + /// 2. Falls back to Docker with default image if binary not found + /// 3. Custom Docker images supported via configuration + /// + /// # Returns + /// + /// Returns a [`ClusterOcamlNodeId`] for referencing this OCaml node + /// in scenarios and peer connections. + /// + /// # Panics + /// + /// Panics if: + /// - No available ports in the configured range + /// - Temporary directory creation fails + /// - OCaml daemon process spawn fails pub fn add_ocaml_node(&mut self, testing_config: OcamlNodeTestingConfig) -> ClusterOcamlNodeId { let node_i = self.ocaml_nodes.len(); + info!( + system_time(); + "Adding OCaml node {} with {} initial peers, block_producer={}", + node_i, + testing_config.initial_peers.len(), + testing_config.block_producer.is_some() + ); + let executable = self.config.ocaml_node_executable(); let mut next_port = || { self.available_ports.next().ok_or_else(|| { @@ -423,19 +654,39 @@ impl Cluster { }; let temp_dir = temp_dir::TempDir::new().expect("failed to create tempdir"); + let libp2p_port = next_port().unwrap(); + let graphql_port = next_port().unwrap(); + let client_port = next_port().unwrap(); + + info!( + system_time(); + "Assigned ports for OCaml node {}: LibP2P={}, GraphQL={}, Client={}", + node_i, + libp2p_port, + graphql_port, + client_port + ); + let node = OcamlNode::start(OcamlNodeConfig { executable, dir: temp_dir, libp2p_keypair_i: self.ocaml_libp2p_keypair_i, - libp2p_port: next_port().unwrap(), - graphql_port: next_port().unwrap(), - client_port: next_port().unwrap(), + libp2p_port, + graphql_port, + client_port, initial_peers: testing_config.initial_peers, daemon_json: testing_config.daemon_json, block_producer: testing_config.block_producer, }) .expect("failed to start ocaml node"); + info!( + system_time(); + "Successfully started OCaml node {} with keypair index {}", + node_i, + self.ocaml_libp2p_keypair_i + ); + self.ocaml_libp2p_keypair_i += 1; self.ocaml_nodes.push(Some(node)); From d36aa40a3c028d04ab93ec26261b2f3892710408 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 23:17:53 +0200 Subject: [PATCH 16/17] Node/testing: logging for OCaml config --- node/testing/src/node/ocaml/config.rs | 205 ++++++++++++++++++++++++-- 1 file changed, 192 insertions(+), 13 deletions(-) diff --git a/node/testing/src/node/ocaml/config.rs b/node/testing/src/node/ocaml/config.rs index e1342cba64..fce6c65327 100644 --- a/node/testing/src/node/ocaml/config.rs +++ b/node/testing/src/node/ocaml/config.rs @@ -1,3 +1,23 @@ +//! OCaml Node Configuration Module +//! +//! This module provides configuration structures and executable management +//! for OCaml Mina nodes in the testing framework. It supports multiple +//! execution modes including local binaries and Docker containers. +//! +//! # Key Components +//! +//! - [`OcamlNodeExecutable`] - Execution method selection (local/Docker) +//! - [`OcamlNodeTestingConfig`] - High-level node configuration +//! - [`OcamlNodeConfig`] - Low-level process configuration +//! - [`DaemonJson`] - Genesis configuration management +//! +//! # Executable Auto-Detection +//! +//! The module automatically detects the best available execution method: +//! 1. Local `mina` binary (preferred) +//! 2. Docker with default image (fallback) +//! 3. Custom Docker images (configurable) + use std::{ ffi::{OsStr, OsString}, fs, @@ -6,13 +26,25 @@ use std::{ str::FromStr, }; -use node::{account::AccountSecretKey, p2p::connection::outgoing::P2pConnectionOutgoingInitOpts}; +use node::{ + account::AccountSecretKey, + core::log::{info, system_time, warn}, + p2p::connection::outgoing::P2pConnectionOutgoingInitOpts, +}; use serde::{Deserialize, Serialize}; +/// High-level configuration for OCaml node testing scenarios. +/// +/// This struct provides the main configuration interface for creating +/// OCaml nodes in test scenarios, abstracting away low-level details +/// like port allocation and process management. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct OcamlNodeTestingConfig { + /// List of initial peer connection targets pub initial_peers: Vec, + /// Genesis ledger configuration (file path or in-memory) pub daemon_json: DaemonJson, + /// Optional block producer secret key pub block_producer: Option, } @@ -53,10 +85,40 @@ pub struct OcamlNodeConfig { pub block_producer: Option, } +/// OCaml node execution methods. +/// +/// Supports multiple ways of running the OCaml Mina daemon, +/// from local binaries to Docker containers with automatic +/// detection and fallback behavior. #[derive(Serialize, Deserialize, Debug, Clone)] pub enum OcamlNodeExecutable { + /// Use locally installed Mina binary + /// + /// # Arguments + /// * `String` - Path to the mina executable + /// + /// # Example + /// ``` + /// OcamlNodeExecutable::Installed("/usr/local/bin/mina".to_string()) + /// ``` Installed(String), + + /// Use specific Docker image + /// + /// # Arguments + /// * `String` - Docker image tag + /// + /// # Example + /// ``` + /// OcamlNodeExecutable::Docker("minaprotocol/mina-daemon:3.0.0".to_string()) + /// ``` Docker(String), + + /// Use default Docker image + /// + /// Falls back to the predefined default image when no local + /// binary is available. See [`OcamlNodeExecutable::DEFAULT_DOCKER_IMAGE`] for the + /// current default. DockerDefault, } @@ -81,17 +143,40 @@ impl OcamlNodeConfig { { match &self.executable { OcamlNodeExecutable::Installed(program) => { + info!(system_time(); "Using local Mina binary: {}", program); let mut cmd = Command::new(program); cmd.envs(envs); cmd } - OcamlNodeExecutable::Docker(tag) => self.docker_run_cmd(tag, envs), + OcamlNodeExecutable::Docker(tag) => { + info!(system_time(); "Using custom Docker image: {}", tag); + self.docker_run_cmd(tag, envs) + } OcamlNodeExecutable::DockerDefault => { + info!( + system_time(); + "Using default Docker image: {}", + OcamlNodeExecutable::DEFAULT_DOCKER_IMAGE + ); self.docker_run_cmd(OcamlNodeExecutable::DEFAULT_DOCKER_IMAGE, envs) } } } + /// Create a Docker run command with proper configuration. + /// + /// Sets up a Docker container with appropriate networking, user mapping, + /// volume mounts, and environment variables for running OCaml Mina daemon. + /// + /// # Arguments + /// * `tag` - Docker image tag to use + /// * `envs` - Environment variables to pass to the container + /// + /// # Docker Configuration + /// - Uses host networking for P2P connectivity + /// - Maps host user ID to avoid permission issues + /// - Mounts node directory for persistent data + /// - Sets working directory to `/tmp` for key generation fn docker_run_cmd(&self, tag: &str, envs: I) -> Command where I: IntoIterator, @@ -104,9 +189,18 @@ impl OcamlNodeConfig { let uid = std::env::var("$UID").unwrap_or_else(|_| "1000".to_owned()); let container_name = OcamlNodeExecutable::docker_container_name(&self.dir); + info!( + system_time(); + "Configuring Docker container: name={}, image={}, uid={}, mount={}", + container_name, + tag, + uid, + dir_path + ); + // set docker opts cmd.arg("run") - .args(["--name".to_owned(), container_name]) + .args(["--name".to_owned(), container_name.clone()]) .args(["--network", "host"]) .args(["--user".to_owned(), format!("{uid}:{uid}")]) .args(["-v".to_owned(), format!("{dir_path}:{dir_path}")]) @@ -116,14 +210,19 @@ impl OcamlNodeConfig { .args(["-w", "/tmp"]); // set docker container envs + let mut env_count = 0; for (key, value) in envs { let arg: OsString = [key.as_ref(), value.as_ref()].join(OsStr::new("=")); cmd.args(["-e".as_ref(), arg.as_os_str()]); + env_count += 1; } + info!(system_time(); "Added {} environment variables to Docker container", env_count); + // set docker image cmd.arg(tag); + info!(system_time(); "Docker command configured for container: {}", container_name); cmd } } @@ -138,40 +237,119 @@ impl OcamlNodeExecutable { format!("mina_testing_ocaml_{}", &path[1..]) } - /// Additional logic for killing the node. + /// Clean up resources when terminating an OCaml node. + /// + /// Handles cleanup logic specific to the execution method: + /// - Local binaries: No additional cleanup needed + /// - Docker containers: Stop and remove the container + /// + /// # Arguments + /// * `tmp_dir` - Temporary directory used by the node pub fn kill(&self, tmp_dir: &temp_dir::TempDir) { match self { - OcamlNodeExecutable::Installed(_) => {} + OcamlNodeExecutable::Installed(program) => { + info!(system_time(); "No additional cleanup needed for local binary: {}", program); + } OcamlNodeExecutable::Docker(_) | OcamlNodeExecutable::DockerDefault => { + let name = Self::docker_container_name(tmp_dir); + let image_info = match self { + OcamlNodeExecutable::Docker(img) => img.clone(), + OcamlNodeExecutable::DockerDefault => Self::DEFAULT_DOCKER_IMAGE.to_string(), + _ => unreachable!(), + }; + + info!( + system_time(); + "Cleaning up Docker container: {} (image: {})", + name, + image_info + ); + // stop container. + info!(system_time(); "Stopping Docker container: {}", name); let mut cmd = Command::new("docker"); - let name = Self::docker_container_name(tmp_dir); - cmd.args(["stop".to_owned(), name]); - let _ = cmd.status(); + cmd.args(["stop".to_owned(), name.clone()]); + match cmd.status() { + Ok(status) if status.success() => { + info!(system_time(); "Successfully stopped Docker container: {}", name); + } + Ok(status) => { + warn!( + system_time(); + "Docker stop command failed for container {}: exit code {:?}", + name, + status.code() + ); + } + Err(e) => { + warn!(system_time(); "Failed to stop Docker container {}: {}", name, e); + } + } // remove container. + info!(system_time(); "Removing Docker container: {}", name); let mut cmd = Command::new("docker"); - let name = Self::docker_container_name(tmp_dir); - cmd.args(["rm".to_owned(), name]); - let _ = cmd.status(); + cmd.args(["rm".to_owned(), name.clone()]); + match cmd.status() { + Ok(status) if status.success() => { + info!(system_time(); "Successfully removed Docker container: {}", name); + } + Ok(status) => { + warn!( + system_time(); + "Docker rm command failed for container {}: exit code {:?}", + name, + status.code() + ); + } + Err(e) => { + warn!(system_time(); "Failed to remove Docker container {}: {}", name, e); + } + } } } } + /// Automatically detect and return the best available OCaml executable. + /// + /// This method implements the auto-detection strategy: + /// 1. First, attempt to use locally installed `mina` binary + /// 2. If not found, fall back to Docker with default image + /// 3. Automatically pull the Docker image if needed + /// + /// # Returns + /// * `Ok(OcamlNodeExecutable)` - Best available execution method + /// * `Err(anyhow::Error)` - No usable execution method found + /// + /// # Docker Fallback + /// When falling back to Docker, this method will automatically + /// pull the default image if not already present locally. pub fn find_working() -> anyhow::Result { let program_name = Self::DEFAULT_MINA_EXECUTABLE; + info!(system_time(); "Attempting to find local Mina binary: {}", program_name); + match Command::new(program_name) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() { - Ok(_) => return Ok(Self::Installed(program_name.to_owned())), + Ok(_) => { + info!(system_time(); "Found working local Mina binary: {}", program_name); + return Ok(Self::Installed(program_name.to_owned())); + } Err(err) => match err.kind() { - std::io::ErrorKind::NotFound => {} + std::io::ErrorKind::NotFound => { + info!(system_time(); "Local Mina binary not found, falling back to Docker"); + } _ => anyhow::bail!("'{program_name}' returned an error: {err}"), }, }; + info!( + system_time(); + "Pulling default Docker image: {}", + Self::DEFAULT_DOCKER_IMAGE + ); let mut cmd = Command::new("docker"); let status = cmd @@ -184,6 +362,7 @@ impl OcamlNodeExecutable { anyhow::bail!("error status pulling ocaml node: {status:?}"); } + info!(system_time(); "Successfully pulled Docker image, using DockerDefault"); Ok(Self::DockerDefault) } } From 5c886fd3f9215fcb71bde0052ae8edcb40682bdf Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Wed, 27 Aug 2025 23:28:57 +0200 Subject: [PATCH 17/17] Node/testing: use logger instead of stdout --- node/testing/src/cluster/mod.rs | 2 +- .../solo_node/basic_connectivity_accept_incoming.rs | 9 +++++---- .../solo_node/basic_connectivity_initial_joining.rs | 10 +++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/node/testing/src/cluster/mod.rs b/node/testing/src/cluster/mod.rs index bc5f838a8a..143ef27907 100644 --- a/node/testing/src/cluster/mod.rs +++ b/node/testing/src/cluster/mod.rs @@ -872,7 +872,7 @@ impl Cluster { let mut i = 0; let total = self.scenario.cur_scenario().steps.len(); loop { - eprintln!("[step]: {i}/{total}"); + info!(system_time(); "Executing step {}/{}", i + 1, total); if !self.exec_next().await? { break Ok(()); } diff --git a/node/testing/src/scenarios/solo_node/basic_connectivity_accept_incoming.rs b/node/testing/src/scenarios/solo_node/basic_connectivity_accept_incoming.rs index 003c6e5e63..0662621106 100644 --- a/node/testing/src/scenarios/solo_node/basic_connectivity_accept_incoming.rs +++ b/node/testing/src/scenarios/solo_node/basic_connectivity_accept_incoming.rs @@ -3,7 +3,10 @@ use std::time::Duration; use libp2p::Multiaddr; -use node::p2p::{connection::outgoing::P2pConnectionOutgoingInitOpts, PeerId}; +use node::{ + core::log::{debug, system_time}, + p2p::{connection::outgoing::P2pConnectionOutgoingInitOpts, PeerId}, +}; use rand::Rng; use crate::{ @@ -99,9 +102,7 @@ impl SoloNodeBasicConnectivityAcceptIncoming { // TODO: the threshold is too small, node cannot connect to many peer before the timeout if ready_peers >= KNOWN_PEERS && known_peers >= KNOWN_PEERS || step >= 1000 { - eprintln!("step: {step}"); - eprintln!("known peers: {known_peers}"); - eprintln!("connected peers: {ready_peers}"); + debug!(system_time(); "Step: {}, known peers: {}, connected peers: {}", step, known_peers, ready_peers); let ocaml_peer_id = if let Some(peer_id) = ocaml_node.as_ref() { *peer_id diff --git a/node/testing/src/scenarios/solo_node/basic_connectivity_initial_joining.rs b/node/testing/src/scenarios/solo_node/basic_connectivity_initial_joining.rs index 4697d8ab91..4fee670914 100644 --- a/node/testing/src/scenarios/solo_node/basic_connectivity_initial_joining.rs +++ b/node/testing/src/scenarios/solo_node/basic_connectivity_initial_joining.rs @@ -4,7 +4,10 @@ use std::{collections::HashMap, time::Duration}; use libp2p::Multiaddr; -use node::p2p::connection::outgoing::P2pConnectionOutgoingInitOpts; +use node::{ + core::log::{debug, system_time}, + p2p::connection::outgoing::P2pConnectionOutgoingInitOpts, +}; use crate::{ hosts, @@ -102,10 +105,7 @@ impl SoloNodeBasicConnectivityInitialJoining { // TODO: the threshold is too small, node cannot connect to many peer before the timeout if ready_peers >= KNOWN_PEERS && known_peers >= KNOWN_PEERS { - eprintln!("step: {step}"); - eprintln!("known peers: {known_peers}"); - eprintln!("connected peers: {ready_peers}"); - eprintln!("success"); + debug!(system_time(); "Step: {}, known peers: {}, connected peers: {}, success", step, known_peers, ready_peers); if let Some(debugger) = runner.debugger() { tokio::time::sleep(Duration::from_secs(10)).await;