From db2043b350cb8f3047866812893b7720f601e4c0 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Thu, 18 Sep 2025 11:56:29 +0200 Subject: [PATCH 01/16] WIP --- Cargo.lock | 7 + Cargo.toml | 1 + justfile | 47 +++++-- scripts/deploy-test-contract | 8 +- test-configs/c0/committee.toml | 1 + test-configs/docker/committee.toml | 1 + test-configs/linux/committee.toml | 42 ++++++ test-configs/linux/net.toml | 38 +++++ test-configs/linux/node_0.toml | 36 +++++ test-configs/linux/node_1.toml | 36 +++++ test-configs/linux/node_2.toml | 36 +++++ test-configs/linux/node_3.toml | 36 +++++ test-configs/linux/node_4.toml | 36 +++++ test-configs/local/committee.toml | 1 + .../nitro-ci-committee/committee.toml | 1 + test-utils/Cargo.toml | 14 ++ test-utils/src/binaries/block-checker.rs | 14 +- test-utils/src/binaries/block-maker.rs | 10 +- test-utils/src/binaries/net-setup.rs | 131 ++++++++++++++++++ test-utils/src/binaries/run-committee.rs | 106 +++++++++++--- test-utils/src/binaries/run.rs | 91 ++++++++++-- test-utils/src/lib.rs | 5 + test-utils/src/net.rs | 38 +++++ test-utils/src/process.rs | 30 ++++ timeboost-config/src/binaries/mkconfig.rs | 6 +- timeboost-config/src/committee.rs | 3 +- timeboost/src/binaries/timeboost.rs | 16 +-- yapper/src/main.rs | 8 +- 28 files changed, 724 insertions(+), 75 deletions(-) create mode 100644 test-configs/linux/committee.toml create mode 100644 test-configs/linux/net.toml create mode 100644 test-configs/linux/node_0.toml create mode 100644 test-configs/linux/node_1.toml create mode 100644 test-configs/linux/node_2.toml create mode 100644 test-configs/linux/node_3.toml create mode 100644 test-configs/linux/node_4.toml create mode 100644 test-utils/src/binaries/net-setup.rs create mode 100644 test-utils/src/net.rs create mode 100644 test-utils/src/process.rs diff --git a/Cargo.lock b/Cargo.lock index 310065bb..635f8afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6594,6 +6594,9 @@ name = "ipnet" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +dependencies = [ + "serde", +] [[package]] name = "iri-string" @@ -11623,16 +11626,20 @@ dependencies = [ "clap", "either", "futures", + "ipnet", + "jiff", "multisig", "prost 0.14.1", "quick_cache", "robusta", "rustix 1.1.2", "sailfish", + "serde", "timeboost", "timeboost-utils", "tokio", "tokio-util", + "toml 0.9.5", "tonic 0.14.2", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 7ac6ea0d..de613321 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,6 +69,7 @@ ethereum_ssz = "0.9.0" futures = { version = "0.3", default-features = false, features = ["alloc"] } generic-array = { version = "0.14.7", features = ["serde", "zeroize"] } http = "1.3.1" +ipnet = { version = "2.11.0", features = ["serde"] } jiff = { version = "0.2", default-features = false, features = ["serde", "std"] } minicbor = { version = "2.1.1", features = ["full"] } nohash-hasher = "0.2" diff --git a/justfile b/justfile index 6fad8258..58ceae67 100644 --- a/justfile +++ b/justfile @@ -32,7 +32,7 @@ build-port-alloc: [private] build-test-utils: - cargo build --release -p test-utils + cargo build --release -p test-utils --all-features #################### ###CHECK COMMANDS### @@ -171,17 +171,42 @@ test-contract-deploy *ARGS: scripts/test-contract-deploy {{ARGS}} test-all: build_release build-test-utils - env RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn target/release/run \ + env RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn \ + target/release/run \ --verbose \ --timeout 120 \ --spawn "1:anvil --port 8545" \ --run "2:sleep 3" \ - --run "3:scripts/deploy-test-contract" \ - --spawn "4:target/release/block-maker --port 55000 --committee test-configs/local/committee.toml" \ - --spawn "4:target/release/yapper --keyset-file test-configs/local/committee.toml" \ - --spawn "5:target/release/run-committee --configs test-configs/local/ --committee 0 --timeboost target/release/timeboost" \ - target/release/block-checker -- \ - --config test-configs/local/node_0.toml \ - --committee test-configs/local/committee.toml \ - --committee-id 0 \ - --blocks 1000 + --run "3:scripts/deploy-test-contract test-configs/local/committee.toml http://localhost:8545" \ + --spawn "4:target/release/block-maker --bind 127.0.0.1:55000 -c test-configs/local/committee.toml" \ + --spawn "4:target/release/yapper -c test-configs/local/committee.toml" \ + --spawn "5:target/release/run-committee -c test-configs/local/ -t target/release/timeboost" \ + target/release/block-checker -- -c test-configs/local -b 1000 + +forward-ipv4 val: build-test-utils + run0 target/release/net-setup system --forward-ipv4 {{val}} + +create-net: build-test-utils + run0 target/release/net-setup create -c test-configs/linux/net.toml + +delete-net: build-test-utils + run0 target/release/net-setup delete -c test-configs/linux/net.toml + +netsim: build_release build-test-utils + env RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn \ + run0 --setenv=PATH --setenv=HOME --setenv=RUST_LOG target/release/run \ + --verbose \ + --timeout 120 \ + --clear-env \ + --env PATH \ + --env HOME \ + --env RUST_LOG \ + --uid `id -u` \ + --gid `id -g` \ + --spawn "1:anvil --host 10.0.1.0 --port 8545" \ + --run "2:sleep 3" \ + --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://10.0.1.0:8545" \ + --spawn "4:target/release/block-maker --bind 10.0.1.0:55000 -c test-configs/linux/committee.toml" \ + --spawn "4:target/release/yapper -c test-configs/linux/committee.toml" \ + --spawn-as-root "5:target/release/run-committee -u `id -u` -g `id -g` -c test-configs/linux/ -t target/release/timeboost" \ + target/release/block-checker -- -c test-configs/linux -b 200 diff --git a/scripts/deploy-test-contract b/scripts/deploy-test-contract index aa40c732..3867ec0b 100755 --- a/scripts/deploy-test-contract +++ b/scripts/deploy-test-contract @@ -2,15 +2,17 @@ set -euo pipefail +COMMITTEE="$1" +URL="$2" + MANAGER_MNEMONIC="attend year erase basket blind adapt stove broccoli isolate unveil acquire category" MANAGER_ACCOUNT_INDEX=0 MANAGER_ADDRESS="0x36561082951eed7ffD59cFD82D70570C57072d02" FAUCET_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" -URL="http://localhost:8545" DEPLOYMENT_FILE=$(mktemp -t timeboost-deployment-XXXXX) # Fund manager account && create deployment file -cast send --value 1ether --private-key "$FAUCET_PRIVATE_KEY" "$MANAGER_ADDRESS" +cast send --value 1ether -r "$URL" --private-key "$FAUCET_PRIVATE_KEY" "$MANAGER_ADDRESS" env RUST_LOG=info cargo run --release --bin deploy -- \ -m "$MANAGER_MNEMONIC" \ -i "$MANAGER_ACCOUNT_INDEX" \ @@ -26,7 +28,7 @@ env RUST_LOG=info cargo run --release --bin register -- \ -i "$MANAGER_ACCOUNT_INDEX" \ -u "$URL" \ -k "$km_addr" \ - -c "test-configs/c0/committee.toml" + -c "$COMMITTEE" # Clean up the temporary deployment file rm -f "$DEPLOYMENT_FILE" diff --git a/test-configs/c0/committee.toml b/test-configs/c0/committee.toml index 73a09462..56b14418 100644 --- a/test-configs/c0/committee.toml +++ b/test-configs/c0/committee.toml @@ -1,3 +1,4 @@ +id = 0 effective_timestamp = "2025-09-01T02:00:00Z" [[members]] diff --git a/test-configs/docker/committee.toml b/test-configs/docker/committee.toml index b43a0c83..d3692f38 100644 --- a/test-configs/docker/committee.toml +++ b/test-configs/docker/committee.toml @@ -1,3 +1,4 @@ +id = 0 effective_timestamp = "2025-09-01T20:00:00Z" [[members]] diff --git a/test-configs/linux/committee.toml b/test-configs/linux/committee.toml new file mode 100644 index 00000000..5b97d3c1 --- /dev/null +++ b/test-configs/linux/committee.toml @@ -0,0 +1,42 @@ +id = 0 +effective_timestamp = "2025-09-01T02:00:00Z" + +[[members]] +signing_key = "eiwaGN1NNaQdbnR9FsjKzUeLghQZsTLPjiL4RcQgfLoX" +dh_key = "AZrLbV37HAGhBWh49JHzup6Wfpu2AAGWGJJnxCDJibiY" +dkg_enc_key = "7PdmfTS45d2hTXB8NcrTmvDwUVBimpYBbrBaGnu3i5Ne65krVfUpbe7bYRHS3AEg7H" +public_address = "10.0.0.1:8000" +internal_api = "10.0.0.1:8003" +http_api = "10.0.0.1:8004" + +[[members]] +signing_key = "vGKKAxVNfkSCdn8qh36nXdSZqyhPq644sQBoeZtcEUCR" +dh_key = "FHTJAk6oyt3jefEp1ZrPEn2MkqRt2LibEFd57AnEUZdb" +dkg_enc_key = "7p1BtEz7WnFMt6Hr28X3Rngqza6i8hRoswhzZRFd6GzgkspLKHBfDocHP8DwzXiNiZ" +public_address = "10.0.0.10:8010" +internal_api = "10.0.0.10:8013" +http_api = "10.0.0.10:8014" + +[[members]] +signing_key = "264jMLf85hfufg4ck97Hw2jiL6i1PHNoGUqxUqfhtssaE" +dh_key = "63eYNKoW2PsWZFhHHj3eZwHTdPE7gEjEDM7gGeDf9Uaj" +dkg_enc_key = "62bnAAbU58zZUcGqy9JKGZRZHkm3g7JZB2DtJGQyChXQBPGvXSS6fF21yoxiVuD1eb" +public_address = "10.0.0.20:8020" +internal_api = "10.0.0.20:8023" +http_api = "10.0.0.20:8024" + +[[members]] +signing_key = "v6UBdLT5BvMhLW7iKv7M2xYeaW2SCAsnZ5PiSg6AaKfA" +dh_key = "Do2GmAexW5MUdD8nToDiBWGbDgk1AwXoxtLTyirDtKQh" +dkg_enc_key = "7aZBFZUEbXxFH9SiGJeUyjzas4mYJ1R13mTPsPeawVU7JFuocfvX9XsRT8qgr17RCe" +public_address = "10.0.0.30:8030" +internal_api = "10.0.0.30:8033" +http_api = "10.0.0.30:8034" + +[[members]] +signing_key = "tV66KknkDH47hRSNzwJtt7Q7EZtxVxQsNnUGoAJdDn6J" +dh_key = "HXaesvEGFiDgrVTix1fKzSLTarFexTZSJD6ymSrF7vPL" +dkg_enc_key = "6R69TzDg3jo1MTex9Uter9XQud458YPpSvXirkYdS295PV81CRvgz5jVWQbkCpZnYV" +public_address = "10.0.0.40:8040" +internal_api = "10.0.0.40:8043" +http_api = "10.0.0.40:8044" diff --git a/test-configs/linux/net.toml b/test-configs/linux/net.toml new file mode 100644 index 00000000..2fae7e80 --- /dev/null +++ b/test-configs/linux/net.toml @@ -0,0 +1,38 @@ +[bridge] +name = "bridge" +cidr = "10.0.1.0/16" + +[[device]] +node = "node_0" +name = "dev1" +cidr = "10.0.0.1/16" +delay = "50ms" +jitter = "50ms" + +[[device]] +node = "node_1" +name = "dev2" +cidr = "10.0.0.10/16" +delay = "50ms" +jitter = "50ms" + +[[device]] +node = "node_2" +name = "dev3" +cidr = "10.0.0.20/16" +delay = "50ms" +jitter = "50ms" + +[[device]] +node = "node_3" +name = "dev4" +cidr = "10.0.0.30/16" +delay = "50ms" +jitter = "50ms" + +[[device]] +node = "node_4" +name = "dev5" +cidr = "10.0.0.40/16" +delay = "50ms" +jitter = "50ms" diff --git a/test-configs/linux/node_0.toml b/test-configs/linux/node_0.toml new file mode 100644 index 00000000..96bccba5 --- /dev/null +++ b/test-configs/linux/node_0.toml @@ -0,0 +1,36 @@ +stamp = "/tmp/timeboost.0.stamp" + +[net.public] +address = "10.0.0.1:8000" +http-api = "10.0.0.1:8004" + +[net.internal] +address = "10.0.0.1:8003" +nitro = "10.0.1.0:55000" + +[keys.signing] +secret = "3hzb3bRzn3dXSV1iEVE6mU4BF2aS725s8AboRxLwULPp" +public = "eiwaGN1NNaQdbnR9FsjKzUeLghQZsTLPjiL4RcQgfLoX" + +[keys.dh] +secret = "BB3zUfFQGfw3sL6bpp1JH1HozK6ehEDmRGoiCpQH62rZ" +public = "AZrLbV37HAGhBWh49JHzup6Wfpu2AAGWGJJnxCDJibiY" + +[keys.dkg] +secret = "BW8gq8MARtDkSJL6daobPtGQm22TKkXdbLNrNGngNGTB" +public = "7PdmfTS45d2hTXB8NcrTmvDwUVBimpYBbrBaGnu3i5Ne65krVfUpbe7bYRHS3AEg7H" + +[chain] +namespace = 10101 + +[chain.parent] +id = 31337 +rpc-url = "http://10.0.1.0:8545/" +ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" +block-tag = "finalized" +key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" + +[espresso] +base-url = "https://query.decaf.testnet.espresso.network/v1/" +websockets-base-url = "wss://query.decaf.testnet.espresso.network/v1/" +max-transaction-size = 1048576 diff --git a/test-configs/linux/node_1.toml b/test-configs/linux/node_1.toml new file mode 100644 index 00000000..0d47e6c1 --- /dev/null +++ b/test-configs/linux/node_1.toml @@ -0,0 +1,36 @@ +stamp = "/tmp/timeboost.1.stamp" + +[net.public] +address = "10.0.0.10:8010" +http-api = "10.0.0.10:8014" + +[net.internal] +address = "10.0.0.10:8013" +nitro = "10.0.1.0:55000" + +[keys.signing] +secret = "FWJzNGvEjFS3h1N1sSMkcvvroWwjT5LQuGkGHu9JMAYs" +public = "vGKKAxVNfkSCdn8qh36nXdSZqyhPq644sQBoeZtcEUCR" + +[keys.dh] +secret = "4hjtciEvuoFVT55nAzvdP9E76r18QwntWwFoeginCGnP" +public = "FHTJAk6oyt3jefEp1ZrPEn2MkqRt2LibEFd57AnEUZdb" + +[keys.dkg] +secret = "ARtqWGmRWrBqZUr4MmiLaPgzjsiKp5USsC9iQNRMZYy4" +public = "7p1BtEz7WnFMt6Hr28X3Rngqza6i8hRoswhzZRFd6GzgkspLKHBfDocHP8DwzXiNiZ" + +[chain] +namespace = 10101 + +[chain.parent] +id = 31337 +rpc-url = "http://10.0.1.0:8545/" +ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" +block-tag = "finalized" +key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" + +[espresso] +base-url = "https://query.decaf.testnet.espresso.network/v1/" +websockets-base-url = "wss://query.decaf.testnet.espresso.network/v1/" +max-transaction-size = 1048576 diff --git a/test-configs/linux/node_2.toml b/test-configs/linux/node_2.toml new file mode 100644 index 00000000..49754442 --- /dev/null +++ b/test-configs/linux/node_2.toml @@ -0,0 +1,36 @@ +stamp = "/tmp/timeboost.2.stamp" + +[net.public] +address = "10.0.0.20:8020" +http-api = "10.0.0.20:8024" + +[net.internal] +address = "10.0.0.20:8023" +nitro = "10.0.1.0:55000" + +[keys.signing] +secret = "2yWTaC6MWvNva97t81cd9QX5qph68NnB1wRVwAChtuGr" +public = "264jMLf85hfufg4ck97Hw2jiL6i1PHNoGUqxUqfhtssaE" + +[keys.dh] +secret = "Fo2nYV4gE9VfoVW9bSySAJ1ZuKT461x6ovZnr3EecCZg" +public = "63eYNKoW2PsWZFhHHj3eZwHTdPE7gEjEDM7gGeDf9Uaj" + +[keys.dkg] +secret = "77r7T3En7NNQvRA81G5hLhJD3VpnigJdkPonX3oAwWkX" +public = "62bnAAbU58zZUcGqy9JKGZRZHkm3g7JZB2DtJGQyChXQBPGvXSS6fF21yoxiVuD1eb" + +[chain] +namespace = 10101 + +[chain.parent] +id = 31337 +rpc-url = "http://10.0.1.0:8545/" +ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" +block-tag = "finalized" +key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" + +[espresso] +base-url = "https://query.decaf.testnet.espresso.network/v1/" +websockets-base-url = "wss://query.decaf.testnet.espresso.network/v1/" +max-transaction-size = 1048576 diff --git a/test-configs/linux/node_3.toml b/test-configs/linux/node_3.toml new file mode 100644 index 00000000..5af84cce --- /dev/null +++ b/test-configs/linux/node_3.toml @@ -0,0 +1,36 @@ +stamp = "/tmp/timeboost.3.stamp" + +[net.public] +address = "10.0.0.30:8030" +http-api = "10.0.0.30:8034" + +[net.internal] +address = "10.0.0.30:8033" +nitro = "10.0.1.0:55000" + +[keys.signing] +secret = "CUpkbkn8bix7ZrbztPKJwu66MRpJrc1Wr2JfdrhetASk" +public = "v6UBdLT5BvMhLW7iKv7M2xYeaW2SCAsnZ5PiSg6AaKfA" + +[keys.dh] +secret = "5KpixkV7czZTDVh7nV7VL1vGk4uf4kjKidDWq34CJx1T" +public = "Do2GmAexW5MUdD8nToDiBWGbDgk1AwXoxtLTyirDtKQh" + +[keys.dkg] +secret = "7vWcVJDAhfSvmtm1L7KZvoD9agx6hyy9FvA75xWpjxK7" +public = "7aZBFZUEbXxFH9SiGJeUyjzas4mYJ1R13mTPsPeawVU7JFuocfvX9XsRT8qgr17RCe" + +[chain] +namespace = 10101 + +[chain.parent] +id = 31337 +rpc-url = "http://10.0.1.0:8545/" +ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" +block-tag = "finalized" +key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" + +[espresso] +base-url = "https://query.decaf.testnet.espresso.network/v1/" +websockets-base-url = "wss://query.decaf.testnet.espresso.network/v1/" +max-transaction-size = 1048576 diff --git a/test-configs/linux/node_4.toml b/test-configs/linux/node_4.toml new file mode 100644 index 00000000..4673392c --- /dev/null +++ b/test-configs/linux/node_4.toml @@ -0,0 +1,36 @@ +stamp = "/tmp/timeboost.4.stamp" + +[net.public] +address = "10.0.0.40:8040" +http-api = "10.0.0.40:8044" + +[net.internal] +address = "10.0.0.40:8043" +nitro = "10.0.1.0:55000" + +[keys.signing] +secret = "6LMMEuoPRCkpDsnxnANCRCBC6JagdCHs2pjNicdmQpQE" +public = "tV66KknkDH47hRSNzwJtt7Q7EZtxVxQsNnUGoAJdDn6J" + +[keys.dh] +secret = "39wAn3bQzpn19oa8CiaNUFd8GekQAJMMuzrbp8Jt3FKz" +public = "HXaesvEGFiDgrVTix1fKzSLTarFexTZSJD6ymSrF7vPL" + +[keys.dkg] +secret = "GFvv2wcQmiGpk5rFp1FGpeUjnVUyZmGM9k8VHb1Jn7EG" +public = "6R69TzDg3jo1MTex9Uter9XQud458YPpSvXirkYdS295PV81CRvgz5jVWQbkCpZnYV" + +[chain] +namespace = 10101 + +[chain.parent] +id = 31337 +rpc-url = "http://10.0.1.0:8545/" +ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" +block-tag = "finalized" +key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" + +[espresso] +base-url = "https://query.decaf.testnet.espresso.network/v1/" +websockets-base-url = "wss://query.decaf.testnet.espresso.network/v1/" +max-transaction-size = 1048576 diff --git a/test-configs/local/committee.toml b/test-configs/local/committee.toml index 73a09462..56b14418 100644 --- a/test-configs/local/committee.toml +++ b/test-configs/local/committee.toml @@ -1,3 +1,4 @@ +id = 0 effective_timestamp = "2025-09-01T02:00:00Z" [[members]] diff --git a/test-configs/nitro-ci-committee/committee.toml b/test-configs/nitro-ci-committee/committee.toml index e58e623b..f058606a 100644 --- a/test-configs/nitro-ci-committee/committee.toml +++ b/test-configs/nitro-ci-committee/committee.toml @@ -1,3 +1,4 @@ +id = 0 effective_timestamp = "2025-09-01T02:00:00Z" [[members]] diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index b76236d0..6952a4ec 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -10,17 +10,21 @@ default = [ "dep:bincode", "dep:bytes", "dep:either", + "dep:jiff", "dep:multisig", "dep:prost", "dep:robusta", "dep:quick_cache", "dep:sailfish", + "dep:serde", "dep:tokio-util", + "dep:toml", "dep:tonic", "dep:timeboost", "dep:timeboost-utils", "ports" ] +netns = ["dep:ipnet"] ports = ["dep:bincode", "dep:tokio-util"] [dependencies] @@ -35,12 +39,16 @@ alloy = { workspace = true, optional = true } bincode = { workspace = true, optional = true } bytes = { workspace = true, optional = true } either = { workspace = true, optional = true } +ipnet = { workspace = true, optional = true } +jiff = { workspace = true, optional = true } multisig = { path = "../multisig", optional = true } prost = { workspace = true, optional = true } quick_cache = { workspace = true, optional = true } robusta = { path = "../robusta", optional = true } sailfish = { path = "../sailfish", optional = true } +serde = { workspace = true, optional = true } tokio-util = { workspace = true, optional = true } +toml = { workspace = true, optional = true } tonic = { workspace = true, optional = true } timeboost = { path = "../timeboost", optional = true } timeboost-utils = { path = "../timeboost-utils", optional = true } @@ -69,3 +77,9 @@ path = "src/binaries/run.rs" [[bin]] name = "run-committee" path = "src/binaries/run-committee.rs" +required-features = ["netns"] + +[[bin]] +name = "net-setup" +path = "src/binaries/net-setup.rs" +required-features = ["netns"] diff --git a/test-utils/src/binaries/block-checker.rs b/test-utils/src/binaries/block-checker.rs index d35f278f..cdcae970 100644 --- a/test-utils/src/binaries/block-checker.rs +++ b/test-utils/src/binaries/block-checker.rs @@ -13,13 +13,7 @@ use tracing::{debug, info}; #[derive(Parser, Debug)] struct Args { #[clap(long, short)] - config: PathBuf, - - #[clap(long)] - committee: PathBuf, - - #[clap(long)] - committee_id: u64, + configs: PathBuf, #[clap(long, short)] blocks: usize, @@ -32,16 +26,16 @@ async fn main() -> Result<()> { let args = Args::parse(); let committees = { - let conf = CommitteeConfig::read(&args.committee).await?; + let conf = CommitteeConfig::read(args.configs.join("committee.toml")).await?; let mems = conf .members .into_iter() .enumerate() .map(|(i, m)| (i as u8, m.signing_key)); - CommitteeVec::<1>::new(Committee::new(args.committee_id, mems)) + CommitteeVec::<1>::new(Committee::new(conf.id, mems)) }; - let node = NodeConfig::read(&args.config).await?; + let node = NodeConfig::read(args.configs.join("node_0.toml")).await?; let conf = Config::builder() .base_url(node.espresso.base_url) diff --git a/test-utils/src/binaries/block-maker.rs b/test-utils/src/binaries/block-maker.rs index f15f4650..c66cd5d4 100644 --- a/test-utils/src/binaries/block-maker.rs +++ b/test-utils/src/binaries/block-maker.rs @@ -1,5 +1,5 @@ use std::convert::Infallible; -use std::net::Ipv4Addr; +use std::net::SocketAddr; use std::path::PathBuf; use std::process::exit; use std::sync::atomic::{AtomicU64, Ordering}; @@ -26,7 +26,7 @@ use tracing::error; #[derive(Parser, Debug)] struct Args { #[clap(long, short)] - port: u16, + bind: SocketAddr, #[clap(long, short)] committee: PathBuf, @@ -51,10 +51,10 @@ impl Service { } } - async fn serve(self, port: u16) -> Result<()> { + async fn serve(self, addr: SocketAddr) -> Result<()> { tonic::transport::Server::builder() .add_service(ForwardApiServer::new(self)) - .serve((Ipv4Addr::UNSPECIFIED, port).into()) + .serve(addr) .await .map_err(From::from) } @@ -126,5 +126,5 @@ async fn main() -> Result<()> { let uri: Uri = format!("http://{}", member.internal_api).parse()?; spawn(deliver(uri, tx.subscribe())); } - Service::new(tx).serve(args.port).await + Service::new(tx).serve(args.bind).await } diff --git a/test-utils/src/binaries/net-setup.rs b/test-utils/src/binaries/net-setup.rs new file mode 100644 index 00000000..57d63302 --- /dev/null +++ b/test-utils/src/binaries/net-setup.rs @@ -0,0 +1,131 @@ +use std::{fs, path::PathBuf}; + +use anyhow::{Result, ensure}; +use clap::Parser; +use ipnet::Ipv4Net; +use jiff::Span; +use test_utils::{ + net::{Config, DeviceConfig}, + process::run_command, +}; + +const TRACE: bool = true; + +#[derive(Debug, Parser)] +enum Command { + System { + #[clap(long, action = clap::ArgAction::Set)] + forward_ipv4: bool, + }, + Create { + #[clap(long, short)] + config: PathBuf, + }, + Delete { + #[clap(long, short)] + config: PathBuf, + }, +} + +fn main() -> Result<()> { + match Command::parse() { + Command::System { forward_ipv4 } => { + let v = u8::from(forward_ipv4); + run_command(TRACE, ["sysctl", "-w", &format!("net.ipv4.ip_forward={v}")])? + } + Command::Create { config } => { + let t = fs::read(&config)?; + let c: Config = toml::from_slice(&t)?; + let b = Bridge::new(&c.bridge.name, c.bridge.cidr); + b.create()?; + for d in c.device { + let dev = Device::new(&d); + dev.create(&b)?; + if !d.delay.is_zero() { + dev.delay(&d.delay, &d.jitter)? + } + } + } + Command::Delete { config } => { + let t = fs::read(&config)?; + let c: Config = toml::from_slice(&t)?; + let b = Bridge::new(&c.bridge.name, c.bridge.cidr); + for d in c.device { + Device::new(&d).delete()? + } + b.delete()? + } + } + Ok(()) +} + +#[derive(Debug)] +struct Device { + space: String, + name: String, + dev: String, + cidr: Ipv4Net, +} + +impl Device { + fn new(cfg: &DeviceConfig) -> Device { + Self { + space: cfg.namespace(), + dev: cfg.device(), + cidr: cfg.cidr, + name: cfg.name.clone(), + } + } + + #[rustfmt::skip] + fn create(&self, b: &Bridge) -> Result<()> { + ensure!(b.net.contains(&self.cidr)); + run_command(TRACE, ["ip", "netns", "add", &self.space])?; + run_command(TRACE, ["ip", "link", "add", &self.dev, "type", "veth", "peer", "name", &self.name])?; + run_command(TRACE, ["ip", "link", "set", &self.name, "up"])?; + run_command(TRACE, ["ip", "link", "set", &self.name, "master", &b.name])?; + run_command(TRACE, ["ip", "link", "set", &self.dev, "netns", &self.space])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "addr", "add", &self.cidr.to_string(), "dev", &self.dev])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "link", "set", &self.dev, "up"])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "link", "set", "lo", "up"])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "route", "add", "default", "via", &b.net.addr().to_string()]) + } + + #[rustfmt::skip] + fn delay(&self, delay: &Span, jitter: &Span) -> Result<()> { + let d = format!("{}ms", delay.get_milliseconds()); + let j = format!("{}ms", jitter.get_milliseconds()); + run_command(TRACE, [ + "ip", "netns", "exec", &self.space, + "tc", "qdisc", "add", "dev", &self.dev, "root", "netem", "delay", &d, &j + ]) + } + + fn delete(self) -> Result<()> { + run_command(TRACE, ["ip", "link", "delete", &self.name])?; + run_command(TRACE, ["ip", "netns", "delete", &self.space]) + } +} + +#[derive(Debug)] +struct Bridge { + name: String, + net: Ipv4Net, +} + +#[rustfmt::skip] +impl Bridge { + fn new(name: &str, ip: Ipv4Net) -> Self { + Self { name: name.to_string(), net: ip } + } + + fn create(&self) -> Result<()> { + run_command(TRACE, ["ip", "link", "add", &self.name, "type", "bridge"])?; + run_command(TRACE, ["ip", "addr", "add", &self.net.to_string(), "dev", "bridge"])?; + run_command(TRACE, ["ip", "link", "set", &self.name, "up"]) + } + + fn delete(self) -> Result<()> { + run_command(TRACE, ["ip", "link", "delete", &self.name]) + } +} diff --git a/test-utils/src/binaries/run-committee.rs b/test-utils/src/binaries/run-committee.rs index f9accd24..8284ba47 100644 --- a/test-utils/src/binaries/run-committee.rs +++ b/test-utils/src/binaries/run-committee.rs @@ -1,8 +1,17 @@ -use std::{ffi::OsStr, path::PathBuf}; +use std::collections::BTreeMap; +use std::process::Command as StdCommand; +use std::{ + ffi::OsStr, + path::{Path, PathBuf}, +}; -use anyhow::{Result, bail}; +use anyhow::{Result, bail, ensure}; use clap::Parser; -use tokio::{fs::read_dir, process::Command}; +use test_utils::net::Config; +use tokio::{ + fs::{self, read_dir}, + process::Command, +}; use tokio_util::task::TaskTracker; #[derive(Parser, Debug)] @@ -11,12 +20,15 @@ struct Args { configs: PathBuf, #[clap(long, short)] - committee: u64, + timeboost: PathBuf, #[clap(long, short)] - timeboost: PathBuf, + uid: Option, + + #[clap(long, short)] + gid: Option, - #[clap(long, short, default_value = "/tmp")] + #[clap(long, default_value = "/tmp")] tmp: PathBuf, } @@ -36,30 +48,56 @@ async fn main() -> Result<()> { bail!("{:?} is not a file", args.timeboost) } - let mut commands = Vec::new(); + let mut netconf: Option = None; + let mut commands = BTreeMap::new(); let mut entries = read_dir(&args.configs).await?; while let Some(entry) = entries.next_entry().await? { - if Some(OsStr::new("toml")) != entry.path().extension() { - continue; + match ConfigType::read(&entry.path()) { + ConfigType::Network => { + ensure!(netconf.is_none()); + let bytes = fs::read(&entry.path()).await?; + netconf = Some(toml::from_slice(&bytes)?); + } + ConfigType::Node(name) => { + let mut cmd = StdCommand::new(args.timeboost.as_os_str()); + cmd.arg("--config").arg(entry.path()).arg("--ignore-stamp"); + commands.insert(name.to_string(), cmd); + } + ConfigType::Committee | ConfigType::Unknown => continue, } - if Some(OsStr::new("committee.toml")) == entry.path().file_name() { - continue; + } + + #[cfg(target_os = "linux")] + if let Some(conf) = netconf { + ensure!(conf.device.len() == commands.len()); + for d in conf.device { + let Some(c) = commands.get_mut(&d.node) else { + eprintln!("no command for device {} node {}", d.name, d.node); + continue; + }; + let mut cmd = StdCommand::new("ip"); + cmd.args(["netns", "exec", &d.namespace()]); + if let Some((uid, gid)) = args.uid.zip(args.gid) { + cmd.args([ + "setpriv", + "--reuid", + &uid.to_string(), + "--regid", + &gid.to_string(), + "--clear-groups", + ]); + } + cmd.arg(c.get_program()).args(c.get_args()); + *c = cmd; } - let mut cmd = Command::new(args.timeboost.as_os_str()); - cmd.arg("--committee-id") - .arg(args.committee.to_string()) - .arg("--config") - .arg(entry.path()) - .arg("--ignore-stamp"); - commands.push(cmd); } let tasks = TaskTracker::new(); - for mut cmd in commands { + for cmd in commands.into_values() { tasks.spawn(async move { - let mut child = cmd.spawn()?; + let mut child = Command::from(cmd).spawn()?; child.wait().await }); } @@ -69,3 +107,31 @@ async fn main() -> Result<()> { Ok(()) } + +enum ConfigType<'a> { + Committee, + Node(&'a str), + Network, + Unknown, +} + +impl<'a> ConfigType<'a> { + fn read(p: &'a Path) -> Self { + if p.extension() != Some(OsStr::new("toml")) { + return ConfigType::Unknown; + } + let Some(name) = p.file_stem().and_then(|n| n.to_str()) else { + return ConfigType::Unknown; + }; + if name.starts_with("node") { + return ConfigType::Node(name); + } + if name == "committee" { + return ConfigType::Committee; + } + if name == "net" { + return ConfigType::Network; + } + ConfigType::Unknown + } +} diff --git a/test-utils/src/binaries/run.rs b/test-utils/src/binaries/run.rs index 078096dd..d86018f9 100644 --- a/test-utils/src/binaries/run.rs +++ b/test-utils/src/binaries/run.rs @@ -3,7 +3,7 @@ use std::ops::{Deref, DerefMut}; use std::time::Duration; use std::{ffi::OsStr, process::ExitStatus}; -use anyhow::{Result, anyhow, bail}; +use anyhow::{Result, anyhow, bail, ensure}; use clap::Parser; use futures::FutureExt; use rustix::process::{Pid, Signal, kill_process_group}; @@ -25,6 +25,20 @@ struct Args { #[clap(long, short, value_parser = parse_command_line)] spawn: Vec, + /// Commands to run as root to completion. + #[clap(long, short, value_parser = parse_command_line)] + run_as_root: Vec, + + /// Commands to run concurrently as root user. + #[clap(long, short, value_parser = parse_command_line)] + spawn_as_root: Vec, + + #[clap(long, short)] + env: Vec, + + #[clap(long)] + clear_env: bool, + /// Optional timeout main command in seconds. #[clap(long, short)] timeout: Option, @@ -32,6 +46,14 @@ struct Args { #[clap(long, short)] verbose: bool, + /// Optional user ID to run processes. + #[clap(long)] + uid: Option, + + /// Optional group ID to run processes. + #[clap(long)] + gid: Option, + /// Main command to execute. main: Vec, } @@ -39,37 +61,59 @@ struct Args { #[tokio::main] async fn main() -> Result<()> { let mut args = Args::parse(); + ensure!(!args.main.is_empty()); + args.run_as_root.iter_mut().for_each(|c| c.root = true); args.spawn.iter_mut().for_each(|c| c.sync = false); + args.spawn_as_root.iter_mut().for_each(|c| { + c.root = true; + c.sync = false + }); let mut commands = args.run; + commands.append(&mut args.run_as_root); commands.append(&mut args.spawn); + commands.append(&mut args.spawn_as_root); commands.sort(); let mut term = signal(SignalKind::terminate())?; let mut intr = signal(SignalKind::interrupt())?; let mut helpers = JoinSet::>::new(); - for commandline in commands { + for cmd in commands { + let uid = cmd.root.then_some(0).or(args.uid); + let gid = cmd.root.then_some(0).or(args.gid); if args.verbose { - if commandline.sync { - eprintln!("running command: {}", commandline.args) + if cmd.sync { + eprintln!("running command: {}", cmd.args) } else { - eprintln!("spawning command: {}", commandline.args) + eprintln!("spawning command: {}", cmd.args) } } - let mut pg = ProcessGroup::spawn(commandline.args.split_whitespace())?; - if commandline.sync { + if cmd.sync { + let mut pg = ProcessGroup::spawn( + uid, + gid, + args.clear_env, + &args.env, + cmd.args.split_whitespace(), + )?; let status = select! { s = pg.wait() => s?, _ = term.recv() => return Ok(()), _ = intr.recv() => return Ok(()), }; if !status.success() { - bail!("{:?} failed with {:?}", commandline.args, status.code()); + bail!("{:?} failed with {:?}", cmd.args, status.code()); } } else { + let mut pg = ProcessGroup::spawn( + uid, + gid, + args.clear_env, + &args.env, + cmd.args.split_whitespace(), + )?; helpers.spawn(async move { - let mut pg = ProcessGroup::spawn(commandline.args.split_whitespace())?; let status = pg.wait().await?; Ok(status) }); @@ -80,7 +124,7 @@ async fn main() -> Result<()> { eprintln!("spawning command: {}", args.main.join(" ")) } - let mut main = ProcessGroup::spawn(args.main)?; + let mut main = ProcessGroup::spawn(args.uid, args.gid, args.clear_env, &args.env, args.main)?; let timeout = if let Some(d) = args.timeout { sleep(Duration::from_secs(d)).boxed() @@ -112,6 +156,7 @@ struct Commandline { prio: u8, args: String, sync: bool, + root: bool, } fn parse_command_line(s: &str) -> Result { @@ -120,6 +165,7 @@ fn parse_command_line(s: &str) -> Result { prio: p.parse()?, args: a.to_string(), sync: true, + root: false, }) } @@ -127,7 +173,13 @@ fn parse_command_line(s: &str) -> Result { struct ProcessGroup(Child, Pid); impl ProcessGroup { - fn spawn(it: I) -> Result + fn spawn( + uid: Option, + gid: Option, + clear: bool, + env: &[String], + it: I, + ) -> Result where I: IntoIterator, S: AsRef, @@ -141,6 +193,23 @@ impl ProcessGroup { cmd.arg(a); } cmd.process_group(0); + if let Some(id) = uid { + cmd.uid(id); + } + if let Some(id) = gid { + cmd.gid(id); + } + if clear { + cmd.env_clear(); + for e in env { + match std::env::var(e) { + Ok(v) => { + cmd.env(e, v); + } + Err(err) => eprintln!("error getting env var {e}: {err}"), + } + } + } let child = cmd.spawn()?; let id = child.id().ok_or_else(|| anyhow!("child already exited"))?; let pid = Pid::from_raw(id.try_into()?).ok_or_else(|| anyhow!("invalid pid"))?; diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index e7af0902..1cfe2d0a 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -1,2 +1,7 @@ +pub mod process; + #[cfg(feature = "ports")] pub mod ports; + +#[cfg(feature = "netns")] +pub mod net; diff --git a/test-utils/src/net.rs b/test-utils/src/net.rs new file mode 100644 index 00000000..728eeb6d --- /dev/null +++ b/test-utils/src/net.rs @@ -0,0 +1,38 @@ +use ipnet::Ipv4Net; +use jiff::Span; +use serde::Deserialize; + +#[derive(Deserialize)] +pub struct Config { + pub bridge: BridgeConfig, + pub device: Vec, +} + +#[derive(Deserialize)] +pub struct BridgeConfig { + pub name: String, + pub cidr: Ipv4Net, +} + +#[derive(Deserialize)] +pub struct DeviceConfig { + pub node: String, + pub name: String, + pub cidr: Ipv4Net, + + #[serde(default)] + pub delay: Span, + + #[serde(default)] + pub jitter: Span, +} + +impl DeviceConfig { + pub fn namespace(&self) -> String { + format!("ns-{}", self.name) + } + + pub fn device(&self) -> String { + format!("dev-{}", self.name) + } +} diff --git a/test-utils/src/process.rs b/test-utils/src/process.rs new file mode 100644 index 00000000..751714da --- /dev/null +++ b/test-utils/src/process.rs @@ -0,0 +1,30 @@ +use anyhow::{Result, anyhow, bail}; +use std::{ffi::OsStr, iter::once, process::Command}; + +pub fn run_command(trace: bool, it: I) -> Result<()> +where + I: IntoIterator, + S: AsRef, +{ + let mut args = it.into_iter(); + let exe = args.next().ok_or_else(|| anyhow!("invalid command"))?; + let mut cmd = Command::new(exe); + for a in args { + cmd.arg(a); + } + if trace { + eprintln!( + "> {}", + once(cmd.get_program()) + .chain(cmd.get_args()) + .map(|os| os.to_string_lossy()) + .collect::>() + .join(" ") + ); + } + let status = cmd.status()?; + if !status.success() { + bail!("command not successful, status = {status:?}") + } + Ok(()) +} diff --git a/timeboost-config/src/binaries/mkconfig.rs b/timeboost-config/src/binaries/mkconfig.rs index c4856291..0b6e2011 100644 --- a/timeboost-config/src/binaries/mkconfig.rs +++ b/timeboost-config/src/binaries/mkconfig.rs @@ -9,7 +9,7 @@ use anyhow::{Result, bail}; use ark_std::rand::SeedableRng as _; use clap::{Parser, ValueEnum}; use cliquenet::Address; -use multisig::x25519; +use multisig::{CommitteeId, x25519}; use secp256k1::rand::SeedableRng as _; use timeboost_config::{ChainConfig, ParentChain}; use timeboost_config::{ @@ -33,6 +33,9 @@ struct Args { #[clap(long)] seed: Option, + #[clap(long)] + committee_id: CommitteeId, + /// The effective timestamp for this new committee. /// /// The timestamp format corresponds to RFC 3339, section 5.6, for example: @@ -211,6 +214,7 @@ impl Args { } let committee_config = CommitteeConfig { + id: self.committee_id, effective_timestamp: self.timestamp, members, }; diff --git a/timeboost-config/src/committee.rs b/timeboost-config/src/committee.rs index 4feaaca7..e28734f6 100644 --- a/timeboost-config/src/committee.rs +++ b/timeboost-config/src/committee.rs @@ -2,7 +2,7 @@ use core::fmt; use std::{path::Path, str::FromStr}; use cliquenet::Address; -use multisig::x25519; +use multisig::{CommitteeId, x25519}; use serde::{Deserialize, Serialize}; use timeboost_crypto::prelude::DkgEncKey; @@ -10,6 +10,7 @@ use crate::ConfigError; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct CommitteeConfig { + pub id: CommitteeId, pub effective_timestamp: jiff::Timestamp, #[serde(default)] pub members: Vec, diff --git a/timeboost/src/binaries/timeboost.rs b/timeboost/src/binaries/timeboost.rs index a06b864c..8de33668 100644 --- a/timeboost/src/binaries/timeboost.rs +++ b/timeboost/src/binaries/timeboost.rs @@ -28,10 +28,6 @@ struct Cli { #[clap(long)] config: PathBuf, - /// CommitteeId for the committee in which this member belongs to - #[clap(long, short)] - committee_id: CommitteeId, - /// Ignore any existing stamp file and start from genesis. #[clap(long, default_value_t = false)] ignore_stamp: bool, @@ -88,13 +84,15 @@ async fn main() -> Result<()> { let contract = KeyManager::new(node_config.chain.parent.key_manager_contract, &provider); + let committee_id = CommitteeId::from(contract.currentCommitteeId().call().await?); + let members: Vec = contract - .getCommitteeById(cli.committee_id.into()) + .getCommitteeById(committee_id.into()) .call() .await? .members; - info!(label = %sign_keypair.public_key(), committee_id = %cli.committee_id, "committee info synced"); + info!(label = %sign_keypair.public_key(), %committee_id, "committee info synced"); let peer_hosts_and_keys = members .iter() @@ -133,7 +131,7 @@ async fn main() -> Result<()> { let sailfish_committee = { let c = Committee::new( - cli.committee_id, + committee_id, sailfish_peer_hosts_and_keys .iter() .enumerate() @@ -144,7 +142,7 @@ async fn main() -> Result<()> { let decrypt_committee = { let c = Committee::new( - cli.committee_id, + committee_id, decrypt_peer_hosts_and_keys .iter() .enumerate() @@ -155,7 +153,7 @@ async fn main() -> Result<()> { let certifier_committee = { let c = Committee::new( - cli.committee_id, + committee_id, certifier_peer_hosts_and_keys .iter() .enumerate() diff --git a/yapper/src/main.rs b/yapper/src/main.rs index 08f93243..83be1d8f 100644 --- a/yapper/src/main.rs +++ b/yapper/src/main.rs @@ -31,8 +31,8 @@ struct Cli { /// Path to file containing the committee member's public info. /// /// The file contains backend urls and public key material. - #[clap(long)] - keyset_file: PathBuf, + #[clap(long, short)] + config: PathBuf, /// Specify how many transactions per second to send to each node #[clap(long, short, default_value_t = 100)] @@ -56,9 +56,9 @@ async fn main() -> Result<()> { let cli = Cli::parse(); // Unpack the keyset file which has the urls - let keyset = CommitteeConfig::read(&cli.keyset_file) + let keyset = CommitteeConfig::read(&cli.config) .await - .with_context(|| format!("opening the keyfile at path {:?}", cli.keyset_file))?; + .with_context(|| format!("opening the config at path {:?}", cli.config))?; let mut addresses = Vec::new(); for node in keyset.members { From 582113faeca3ce21d6cd140f078211a281818e92 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Thu, 18 Sep 2025 18:19:27 +0200 Subject: [PATCH 02/16] Use sudo on CI. --- .github/workflows/build-and-test.yml | 29 ++++++++++++++++++- justfile | 42 +++++++++++++++------------- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 046200c3..a03def36 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -142,6 +142,34 @@ jobs: - name: Test with block-maker run: just test-all + netsim: + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + fetch-depth: 0 + - name: Install protobuf compiler + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + - name: Install Just + run: | + wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz + tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just + sudo cp just /usr/bin/just + - name: Install rustfmt for nightly + run: rustup component add --toolchain nightly rustfmt + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + - name: Enable ip4 forwarding + run: just forward-ipv4 true + - name: Create network namespaces + run: just create-net + - name: Run network simulation + run: just netsim + contracts: runs-on: ubuntu-latest steps: @@ -201,4 +229,3 @@ jobs: - name: Docker Logs Sequencer 2 if: failure() run: docker logs test-node-sequencer_b-1 - diff --git a/justfile b/justfile index 58ceae67..b04ccfd2 100644 --- a/justfile +++ b/justfile @@ -4,6 +4,8 @@ export RUSTDOCFLAGS := '-D warnings' LOG_LEVELS := "RUST_LOG=timeboost=debug,sailfish=debug,cliquenet=debug,tests=debug" +run_as_root := if env("CI", "false") == "true" { "sudo" } else { "run0" } + #################### ###BUILD COMMANDS### #################### @@ -184,29 +186,29 @@ test-all: build_release build-test-utils target/release/block-checker -- -c test-configs/local -b 1000 forward-ipv4 val: build-test-utils - run0 target/release/net-setup system --forward-ipv4 {{val}} + {{run_as_root}} target/release/net-setup system --forward-ipv4 {{val}} create-net: build-test-utils - run0 target/release/net-setup create -c test-configs/linux/net.toml + {{run_as_root}} target/release/net-setup create -c test-configs/linux/net.toml delete-net: build-test-utils - run0 target/release/net-setup delete -c test-configs/linux/net.toml + {{run_as_root}} target/release/net-setup delete -c test-configs/linux/net.toml netsim: build_release build-test-utils - env RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn \ - run0 --setenv=PATH --setenv=HOME --setenv=RUST_LOG target/release/run \ - --verbose \ - --timeout 120 \ - --clear-env \ - --env PATH \ - --env HOME \ - --env RUST_LOG \ - --uid `id -u` \ - --gid `id -g` \ - --spawn "1:anvil --host 10.0.1.0 --port 8545" \ - --run "2:sleep 3" \ - --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://10.0.1.0:8545" \ - --spawn "4:target/release/block-maker --bind 10.0.1.0:55000 -c test-configs/linux/committee.toml" \ - --spawn "4:target/release/yapper -c test-configs/linux/committee.toml" \ - --spawn-as-root "5:target/release/run-committee -u `id -u` -g `id -g` -c test-configs/linux/ -t target/release/timeboost" \ - target/release/block-checker -- -c test-configs/linux -b 200 + env RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn \ + {{run_as_root}} --setenv=PATH --setenv=HOME --setenv=RUST_LOG target/release/run \ + --verbose \ + --timeout 120 \ + --clear-env \ + --env PATH \ + --env HOME \ + --env RUST_LOG \ + --uid `id -u` \ + --gid `id -g` \ + --spawn "1:anvil --host 10.0.1.0 --port 8545" \ + --run "2:sleep 3" \ + --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://10.0.1.0:8545" \ + --spawn "4:target/release/block-maker --bind 10.0.1.0:55000 -c test-configs/linux/committee.toml" \ + --spawn "4:target/release/yapper -c test-configs/linux/committee.toml" \ + --spawn-as-root "5:target/release/run-committee -u `id -u` -g `id -g` -c test-configs/linux/ -t target/release/timeboost" \ + target/release/block-checker -- -c test-configs/linux -b 200 From be54f98ca1bff96c2cf6022441d5ad4abce2af73 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Thu, 18 Sep 2025 18:22:55 +0200 Subject: [PATCH 03/16] Fix run-timeboost-demo. --- scripts/run-timeboost-demo | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/run-timeboost-demo b/scripts/run-timeboost-demo index e2d8102e..688dc940 100755 --- a/scripts/run-timeboost-demo +++ b/scripts/run-timeboost-demo @@ -115,7 +115,6 @@ pids=() i=0 for f in "$config_dir"/node_*.toml; do cmd=(target/release/timeboost - --committee-id 0 --committee "$config_dir/committee.toml" --config "$f" --until $rounds From beca8c01281aa8819cc709888c5374eca8bcf9cf Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Thu, 18 Sep 2025 18:30:10 +0200 Subject: [PATCH 04/16] Use `taiki-e/install-action@just` --- .github/workflows/build-and-test.yml | 48 +++++----------------------- .github/workflows/push-docker.yml | 2 +- 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index a03def36..e77e9274 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -26,11 +26,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry @@ -59,11 +55,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry @@ -82,11 +74,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry @@ -104,11 +92,7 @@ jobs: submodules: recursive fetch-depth: 0 - uses: docker/setup-buildx-action@v3 - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Build Docker run: just build_docker @@ -124,11 +108,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry @@ -154,11 +134,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry @@ -177,11 +153,7 @@ jobs: with: submodules: recursive fetch-depth: 0 - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry @@ -201,11 +173,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Install Just - run: | - wget https://github.com/casey/just/releases/download/1.14.0/just-1.14.0-x86_64-unknown-linux-musl.tar.gz - tar -vxf just-1.14.0-x86_64-unknown-linux-musl.tar.gz just - sudo cp just /usr/bin/just + - uses: taiki-e/install-action@just - name: Install rustfmt for nightly run: rustup component add --toolchain nightly rustfmt - name: Install Foundry diff --git a/.github/workflows/push-docker.yml b/.github/workflows/push-docker.yml index c7befdeb..b6b162ae 100644 --- a/.github/workflows/push-docker.yml +++ b/.github/workflows/push-docker.yml @@ -47,4 +47,4 @@ jobs: push: true platforms: linux/amd64 file: ./docker/timeboost.Dockerfile - tags: ghcr.io/espressosystems/timeboost:${{ env.TAG }} \ No newline at end of file + tags: ghcr.io/espressosystems/timeboost:${{ env.TAG }} From 109ff91077318c8bdbdb893fd2b9da64d1d89236 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Thu, 18 Sep 2025 18:36:58 +0200 Subject: [PATCH 05/16] Simplify. --- .github/workflows/build-and-test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index e77e9274..7fab3416 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -139,12 +139,11 @@ jobs: run: rustup component add --toolchain nightly rustfmt - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - - name: Enable ip4 forwarding - run: just forward-ipv4 true - - name: Create network namespaces - run: just create-net - name: Run network simulation - run: just netsim + run: | + just forward-ipv4 true + just create-net + just netsim contracts: runs-on: ubuntu-latest From 23b61477c77c10055b0af2968bbc1d9f7fdf059d Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 13:00:03 +0200 Subject: [PATCH 06/16] Add NAT settings. --- scripts/deploy-test-contract | 4 ++-- test-configs/linux/net.toml | 5 +++++ test-utils/src/binaries/net-setup.rs | 19 +++++++++++++++---- test-utils/src/net.rs | 8 ++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/scripts/deploy-test-contract b/scripts/deploy-test-contract index 3867ec0b..318c8006 100755 --- a/scripts/deploy-test-contract +++ b/scripts/deploy-test-contract @@ -13,7 +13,7 @@ DEPLOYMENT_FILE=$(mktemp -t timeboost-deployment-XXXXX) # Fund manager account && create deployment file cast send --value 1ether -r "$URL" --private-key "$FAUCET_PRIVATE_KEY" "$MANAGER_ADDRESS" -env RUST_LOG=info cargo run --release --bin deploy -- \ +env RUST_LOG=info target/release/deploy \ -m "$MANAGER_MNEMONIC" \ -i "$MANAGER_ACCOUNT_INDEX" \ -u "$URL" \ @@ -23,7 +23,7 @@ env RUST_LOG=info cargo run --release --bin deploy -- \ km_addr=$(sed -nr 's/^key_manager.*=.*"(.+)"/\1/p' "$DEPLOYMENT_FILE") # Update the contract -env RUST_LOG=info cargo run --release --bin register -- \ +env RUST_LOG=info target/release/register \ -m "$MANAGER_MNEMONIC" \ -i "$MANAGER_ACCOUNT_INDEX" \ -u "$URL" \ diff --git a/test-configs/linux/net.toml b/test-configs/linux/net.toml index 2fae7e80..05eddc1b 100644 --- a/test-configs/linux/net.toml +++ b/test-configs/linux/net.toml @@ -1,3 +1,8 @@ +[nat] +table = "nat" +device = "eth0" +cidr = "10.0.0.0/8" + [bridge] name = "bridge" cidr = "10.0.1.0/16" diff --git a/test-utils/src/binaries/net-setup.rs b/test-utils/src/binaries/net-setup.rs index 57d63302..67307842 100644 --- a/test-utils/src/binaries/net-setup.rs +++ b/test-utils/src/binaries/net-setup.rs @@ -1,3 +1,5 @@ +#![cfg_attr(rustfmt, rustfmt_skip)] + use std::{fs, path::PathBuf}; use anyhow::{Result, ensure}; @@ -45,6 +47,15 @@ fn main() -> Result<()> { dev.delay(&d.delay, &d.jitter)? } } + if let Some(nat) = c.nat { + run_command(TRACE, ["iptables", + "-t", &nat.table, + "-A", "POSTROUTING", + "-s", &nat.cidr.to_string(), + "-o", &nat.device, + "-j", "MASQUERADE" + ])? + } } Command::Delete { config } => { let t = fs::read(&config)?; @@ -53,7 +64,10 @@ fn main() -> Result<()> { for d in c.device { Device::new(&d).delete()? } - b.delete()? + b.delete()?; + if let Some(nat) = c.nat { + run_command(TRACE, ["iptables", "-t", &nat.table, "-F"])? + } } } Ok(()) @@ -77,7 +91,6 @@ impl Device { } } - #[rustfmt::skip] fn create(&self, b: &Bridge) -> Result<()> { ensure!(b.net.contains(&self.cidr)); run_command(TRACE, ["ip", "netns", "add", &self.space])?; @@ -91,7 +104,6 @@ impl Device { run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "route", "add", "default", "via", &b.net.addr().to_string()]) } - #[rustfmt::skip] fn delay(&self, delay: &Span, jitter: &Span) -> Result<()> { let d = format!("{}ms", delay.get_milliseconds()); let j = format!("{}ms", jitter.get_milliseconds()); @@ -113,7 +125,6 @@ struct Bridge { net: Ipv4Net, } -#[rustfmt::skip] impl Bridge { fn new(name: &str, ip: Ipv4Net) -> Self { Self { name: name.to_string(), net: ip } diff --git a/test-utils/src/net.rs b/test-utils/src/net.rs index 728eeb6d..a2738a3c 100644 --- a/test-utils/src/net.rs +++ b/test-utils/src/net.rs @@ -6,6 +6,7 @@ use serde::Deserialize; pub struct Config { pub bridge: BridgeConfig, pub device: Vec, + pub nat: Option, } #[derive(Deserialize)] @@ -27,6 +28,13 @@ pub struct DeviceConfig { pub jitter: Span, } +#[derive(Deserialize)] +pub struct NatConfig { + pub table: String, + pub device: String, + pub cidr: Ipv4Net, +} + impl DeviceConfig { pub fn namespace(&self) -> String { format!("ns-{}", self.name) From 012d30a87850a62053cfb2a660c97d2f7f937921 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 13:09:01 +0200 Subject: [PATCH 07/16] Fix sudo on CI. --- .github/workflows/build-and-test.yml | 5 +-- justfile | 58 ++++++++++++++++------------ 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7fab3416..d2b8ebfd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -140,10 +140,7 @@ jobs: - name: Install Foundry uses: foundry-rs/foundry-toolchain@v1 - name: Run network simulation - run: | - just forward-ipv4 true - just create-net - just netsim + run: just forward-ipv4 true create-net netsim contracts: runs-on: ubuntu-latest diff --git a/justfile b/justfile index b04ccfd2..8797e7ba 100644 --- a/justfile +++ b/justfile @@ -1,10 +1,7 @@ -set export - export RUSTDOCFLAGS := '-D warnings' -LOG_LEVELS := "RUST_LOG=timeboost=debug,sailfish=debug,cliquenet=debug,tests=debug" - -run_as_root := if env("CI", "false") == "true" { "sudo" } else { "run0" } +log_levels := "RUST_LOG=timeboost=debug,sailfish=debug,cliquenet=debug,tests=debug" +run_as_root := if env("CI", "") == "true" { "sudo" } else { "run0" } #################### ###BUILD COMMANDS### @@ -156,10 +153,10 @@ test-contracts: build-contracts forge test test_ci *ARGS: build-port-alloc - env {{LOG_LEVELS}} NO_COLOR=1 target/release/run \ + env {{log_levels}} NO_COLOR=1 target/release/run \ --spawn target/release/port-alloc \ cargo nextest run -- --workspace {{ARGS}} - env {{LOG_LEVELS}} NO_COLOR=1 cargo test --doc {{ARGS}} + env {{log_levels}} NO_COLOR=1 cargo test --doc {{ARGS}} test-individually: build-port-alloc @for pkg in $(cargo metadata --no-deps --format-version 1 | jq -r '.packages[].name'); do \ @@ -185,30 +182,43 @@ test-all: build_release build-test-utils --spawn "5:target/release/run-committee -c test-configs/local/ -t target/release/timeboost" \ target/release/block-checker -- -c test-configs/local -b 1000 +[linux] forward-ipv4 val: build-test-utils {{run_as_root}} target/release/net-setup system --forward-ipv4 {{val}} +[linux] create-net: build-test-utils {{run_as_root}} target/release/net-setup create -c test-configs/linux/net.toml +[linux] delete-net: build-test-utils {{run_as_root}} target/release/net-setup delete -c test-configs/linux/net.toml +[linux] netsim: build_release build-test-utils - env RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn \ - {{run_as_root}} --setenv=PATH --setenv=HOME --setenv=RUST_LOG target/release/run \ - --verbose \ - --timeout 120 \ - --clear-env \ - --env PATH \ - --env HOME \ - --env RUST_LOG \ - --uid `id -u` \ - --gid `id -g` \ - --spawn "1:anvil --host 10.0.1.0 --port 8545" \ - --run "2:sleep 3" \ - --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://10.0.1.0:8545" \ - --spawn "4:target/release/block-maker --bind 10.0.1.0:55000 -c test-configs/linux/committee.toml" \ - --spawn "4:target/release/yapper -c test-configs/linux/committee.toml" \ - --spawn-as-root "5:target/release/run-committee -u `id -u` -g `id -g` -c test-configs/linux/ -t target/release/timeboost" \ - target/release/block-checker -- -c test-configs/linux -b 200 + #!/usr/bin/env bash + set -eo pipefail + function run_as_root { + if [ "$CI" == "true" ]; then + sudo "$@" + else + run0 --setenv=PATH --setenv=HOME --setenv=RUST_LOG "$@" + fi + } + export RUST_LOG=timeboost_builder::submit=debug,block_checker=info,warn + run_as_root target/release/run \ + --verbose \ + --timeout 120 \ + --clear-env \ + --env PATH \ + --env HOME \ + --env RUST_LOG \ + --uid $(id -u) \ + --gid $(id -g) \ + --spawn "1:anvil --host 10.0.1.0 --port 8545" \ + --run "2:sleep 3" \ + --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://10.0.1.0:8545" \ + --spawn "4:target/release/block-maker --bind 10.0.1.0:55000 -c test-configs/linux/committee.toml" \ + --spawn "4:target/release/yapper -c test-configs/linux/committee.toml" \ + --spawn-as-root "5:target/release/run-committee -u $(id -u) -g $(id -g) -c test-configs/linux/ -t target/release/timeboost" \ + target/release/block-checker -- -c test-configs/linux -b 200 From 084021f57e736d236ffecff66c39bc0448308fab Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 13:36:13 +0200 Subject: [PATCH 08/16] Some fixes. --- justfile | 2 +- scripts/run-timeboost-demo | 3 ++- test-utils/src/binaries/run-committee.rs | 9 ++++++++- timeboost/src/binaries/timeboost.rs | 16 +++++++++------- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/justfile b/justfile index 8797e7ba..a68f5683 100644 --- a/justfile +++ b/justfile @@ -200,7 +200,7 @@ netsim: build_release build-test-utils set -eo pipefail function run_as_root { if [ "$CI" == "true" ]; then - sudo "$@" + sudo --preserve-env=RUST_LOG "$@" else run0 --setenv=PATH --setenv=HOME --setenv=RUST_LOG "$@" fi diff --git a/scripts/run-timeboost-demo b/scripts/run-timeboost-demo index 688dc940..d456e848 100755 --- a/scripts/run-timeboost-demo +++ b/scripts/run-timeboost-demo @@ -115,6 +115,7 @@ pids=() i=0 for f in "$config_dir"/node_*.toml; do cmd=(target/release/timeboost + --committee-id 0 --committee "$config_dir/committee.toml" --config "$f" --until $rounds @@ -142,7 +143,7 @@ done if $yapper; then cmd=(target/release/yapper --tps $tps - --keyset-file "$config_dir/committee.toml") + --config "$config_dir/committee.toml") if [ $nitro_url ]; then cmd+=(--nitro-url "$nitro_url") fi diff --git a/test-utils/src/binaries/run-committee.rs b/test-utils/src/binaries/run-committee.rs index 8284ba47..9df012bf 100644 --- a/test-utils/src/binaries/run-committee.rs +++ b/test-utils/src/binaries/run-committee.rs @@ -8,6 +8,7 @@ use std::{ use anyhow::{Result, bail, ensure}; use clap::Parser; use test_utils::net::Config; +use timeboost::config::CommitteeConfig; use tokio::{ fs::{self, read_dir}, process::Command, @@ -48,6 +49,8 @@ async fn main() -> Result<()> { bail!("{:?} is not a file", args.timeboost) } + let committee = CommitteeConfig::read(args.configs.join("committee.toml")).await?; + let mut netconf: Option = None; let mut commands = BTreeMap::new(); let mut entries = read_dir(&args.configs).await?; @@ -61,7 +64,11 @@ async fn main() -> Result<()> { } ConfigType::Node(name) => { let mut cmd = StdCommand::new(args.timeboost.as_os_str()); - cmd.arg("--config").arg(entry.path()).arg("--ignore-stamp"); + cmd.arg("--committee-id") + .arg(committee.id.to_string()) + .arg("--config") + .arg(entry.path()) + .arg("--ignore-stamp"); commands.insert(name.to_string(), cmd); } ConfigType::Committee | ConfigType::Unknown => continue, diff --git a/timeboost/src/binaries/timeboost.rs b/timeboost/src/binaries/timeboost.rs index 8de33668..a06b864c 100644 --- a/timeboost/src/binaries/timeboost.rs +++ b/timeboost/src/binaries/timeboost.rs @@ -28,6 +28,10 @@ struct Cli { #[clap(long)] config: PathBuf, + /// CommitteeId for the committee in which this member belongs to + #[clap(long, short)] + committee_id: CommitteeId, + /// Ignore any existing stamp file and start from genesis. #[clap(long, default_value_t = false)] ignore_stamp: bool, @@ -84,15 +88,13 @@ async fn main() -> Result<()> { let contract = KeyManager::new(node_config.chain.parent.key_manager_contract, &provider); - let committee_id = CommitteeId::from(contract.currentCommitteeId().call().await?); - let members: Vec = contract - .getCommitteeById(committee_id.into()) + .getCommitteeById(cli.committee_id.into()) .call() .await? .members; - info!(label = %sign_keypair.public_key(), %committee_id, "committee info synced"); + info!(label = %sign_keypair.public_key(), committee_id = %cli.committee_id, "committee info synced"); let peer_hosts_and_keys = members .iter() @@ -131,7 +133,7 @@ async fn main() -> Result<()> { let sailfish_committee = { let c = Committee::new( - committee_id, + cli.committee_id, sailfish_peer_hosts_and_keys .iter() .enumerate() @@ -142,7 +144,7 @@ async fn main() -> Result<()> { let decrypt_committee = { let c = Committee::new( - committee_id, + cli.committee_id, decrypt_peer_hosts_and_keys .iter() .enumerate() @@ -153,7 +155,7 @@ async fn main() -> Result<()> { let certifier_committee = { let c = Committee::new( - committee_id, + cli.committee_id, certifier_peer_hosts_and_keys .iter() .enumerate() From a55f5f4a23c544ba72910d8682e7c83a8843a229 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 14:21:55 +0200 Subject: [PATCH 09/16] More CI fixes. --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index a68f5683..7a82b383 100644 --- a/justfile +++ b/justfile @@ -200,7 +200,7 @@ netsim: build_release build-test-utils set -eo pipefail function run_as_root { if [ "$CI" == "true" ]; then - sudo --preserve-env=RUST_LOG "$@" + sudo --preserve-env=PATH,HOME,RUST_LOG "$@" else run0 --setenv=PATH --setenv=HOME --setenv=RUST_LOG "$@" fi From 6efdfd00f694bb717855cd322efe197946de78b5 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 16:57:35 +0200 Subject: [PATCH 10/16] Test --- .github/workflows/build-and-test.yml | 2 ++ test-configs/linux/net.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index d2b8ebfd..b157e35d 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -130,6 +130,8 @@ jobs: with: submodules: recursive fetch-depth: 0 + - name: Show net devices + run: ip -o addr show scope global - name: Install protobuf compiler run: | sudo apt-get update diff --git a/test-configs/linux/net.toml b/test-configs/linux/net.toml index 05eddc1b..3e98fbdb 100644 --- a/test-configs/linux/net.toml +++ b/test-configs/linux/net.toml @@ -1,6 +1,6 @@ [nat] table = "nat" -device = "eth0" +device = "enp0s3" cidr = "10.0.0.0/8" [bridge] From 726de00e0bdeb1dccf5e48a9c8726da90cff926d Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 18:24:15 +0200 Subject: [PATCH 11/16] Revert net.toml change. --- test-configs/linux/net.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-configs/linux/net.toml b/test-configs/linux/net.toml index 3e98fbdb..05eddc1b 100644 --- a/test-configs/linux/net.toml +++ b/test-configs/linux/net.toml @@ -1,6 +1,6 @@ [nat] table = "nat" -device = "enp0s3" +device = "eth0" cidr = "10.0.0.0/8" [bridge] From 5da36f58ddc2b28e783c0e033be93a51f87e1476 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 18:41:32 +0200 Subject: [PATCH 12/16] Fix merge error. --- test-utils/src/binaries/run-committee.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test-utils/src/binaries/run-committee.rs b/test-utils/src/binaries/run-committee.rs index 44584a11..d49847cd 100644 --- a/test-utils/src/binaries/run-committee.rs +++ b/test-utils/src/binaries/run-committee.rs @@ -20,8 +20,6 @@ struct Args { #[clap(long, short)] configs: PathBuf, - #[clap(long, short)] - #[clap(long, short, default_value = "target/release/timeboost")] timeboost: PathBuf, From 6a698cdaa52754e5c516796080c8aa3fc5ddb459 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 18:42:16 +0200 Subject: [PATCH 13/16] Use different IP address. --- justfile | 6 +++--- test-configs/linux/committee.toml | 30 +++++++++++++++--------------- test-configs/linux/net.toml | 14 +++++++------- test-configs/linux/node_0.toml | 12 ++++++------ test-configs/linux/node_1.toml | 12 ++++++------ test-configs/linux/node_2.toml | 12 ++++++------ test-configs/linux/node_3.toml | 12 ++++++------ test-configs/linux/node_4.toml | 12 ++++++------ 8 files changed, 55 insertions(+), 55 deletions(-) diff --git a/justfile b/justfile index 39390806..03d2a77e 100644 --- a/justfile +++ b/justfile @@ -265,10 +265,10 @@ netsim: build_release build-test-utils --env RUST_LOG \ --uid $(id -u) \ --gid $(id -g) \ - --spawn "1:anvil --host 10.0.1.0 --port 8545" \ + --spawn "1:anvil --host 11.0.1.0 --port 8545" \ --run "2:sleep 3" \ - --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://10.0.1.0:8545" \ - --spawn "4:target/release/block-maker --bind 10.0.1.0:55000 -c test-configs/linux/committee.toml" \ + --run "3:scripts/deploy-test-contract test-configs/linux/committee.toml http://11.0.1.0:8545" \ + --spawn "4:target/release/block-maker --bind 11.0.1.0:55000 -c test-configs/linux/committee.toml" \ --spawn "4:target/release/yapper -c test-configs/linux/committee.toml" \ --spawn-as-root "5:target/release/run-committee -u $(id -u) -g $(id -g) -c test-configs/linux/" \ target/release/block-checker -- -c test-configs/linux -b 200 diff --git a/test-configs/linux/committee.toml b/test-configs/linux/committee.toml index 5b97d3c1..636dd228 100644 --- a/test-configs/linux/committee.toml +++ b/test-configs/linux/committee.toml @@ -5,38 +5,38 @@ effective_timestamp = "2025-09-01T02:00:00Z" signing_key = "eiwaGN1NNaQdbnR9FsjKzUeLghQZsTLPjiL4RcQgfLoX" dh_key = "AZrLbV37HAGhBWh49JHzup6Wfpu2AAGWGJJnxCDJibiY" dkg_enc_key = "7PdmfTS45d2hTXB8NcrTmvDwUVBimpYBbrBaGnu3i5Ne65krVfUpbe7bYRHS3AEg7H" -public_address = "10.0.0.1:8000" -internal_api = "10.0.0.1:8003" -http_api = "10.0.0.1:8004" +public_address = "11.0.0.1:8000" +internal_api = "11.0.0.1:8003" +http_api = "11.0.0.1:8004" [[members]] signing_key = "vGKKAxVNfkSCdn8qh36nXdSZqyhPq644sQBoeZtcEUCR" dh_key = "FHTJAk6oyt3jefEp1ZrPEn2MkqRt2LibEFd57AnEUZdb" dkg_enc_key = "7p1BtEz7WnFMt6Hr28X3Rngqza6i8hRoswhzZRFd6GzgkspLKHBfDocHP8DwzXiNiZ" -public_address = "10.0.0.10:8010" -internal_api = "10.0.0.10:8013" -http_api = "10.0.0.10:8014" +public_address = "11.0.0.10:8010" +internal_api = "11.0.0.10:8013" +http_api = "11.0.0.10:8014" [[members]] signing_key = "264jMLf85hfufg4ck97Hw2jiL6i1PHNoGUqxUqfhtssaE" dh_key = "63eYNKoW2PsWZFhHHj3eZwHTdPE7gEjEDM7gGeDf9Uaj" dkg_enc_key = "62bnAAbU58zZUcGqy9JKGZRZHkm3g7JZB2DtJGQyChXQBPGvXSS6fF21yoxiVuD1eb" -public_address = "10.0.0.20:8020" -internal_api = "10.0.0.20:8023" -http_api = "10.0.0.20:8024" +public_address = "11.0.0.20:8020" +internal_api = "11.0.0.20:8023" +http_api = "11.0.0.20:8024" [[members]] signing_key = "v6UBdLT5BvMhLW7iKv7M2xYeaW2SCAsnZ5PiSg6AaKfA" dh_key = "Do2GmAexW5MUdD8nToDiBWGbDgk1AwXoxtLTyirDtKQh" dkg_enc_key = "7aZBFZUEbXxFH9SiGJeUyjzas4mYJ1R13mTPsPeawVU7JFuocfvX9XsRT8qgr17RCe" -public_address = "10.0.0.30:8030" -internal_api = "10.0.0.30:8033" -http_api = "10.0.0.30:8034" +public_address = "11.0.0.30:8030" +internal_api = "11.0.0.30:8033" +http_api = "11.0.0.30:8034" [[members]] signing_key = "tV66KknkDH47hRSNzwJtt7Q7EZtxVxQsNnUGoAJdDn6J" dh_key = "HXaesvEGFiDgrVTix1fKzSLTarFexTZSJD6ymSrF7vPL" dkg_enc_key = "6R69TzDg3jo1MTex9Uter9XQud458YPpSvXirkYdS295PV81CRvgz5jVWQbkCpZnYV" -public_address = "10.0.0.40:8040" -internal_api = "10.0.0.40:8043" -http_api = "10.0.0.40:8044" +public_address = "11.0.0.40:8040" +internal_api = "11.0.0.40:8043" +http_api = "11.0.0.40:8044" diff --git a/test-configs/linux/net.toml b/test-configs/linux/net.toml index 05eddc1b..10b9b29c 100644 --- a/test-configs/linux/net.toml +++ b/test-configs/linux/net.toml @@ -1,43 +1,43 @@ [nat] table = "nat" device = "eth0" -cidr = "10.0.0.0/8" +cidr = "11.0.0.0/8" [bridge] name = "bridge" -cidr = "10.0.1.0/16" +cidr = "11.0.1.0/16" [[device]] node = "node_0" name = "dev1" -cidr = "10.0.0.1/16" +cidr = "11.0.0.1/16" delay = "50ms" jitter = "50ms" [[device]] node = "node_1" name = "dev2" -cidr = "10.0.0.10/16" +cidr = "11.0.0.10/16" delay = "50ms" jitter = "50ms" [[device]] node = "node_2" name = "dev3" -cidr = "10.0.0.20/16" +cidr = "11.0.0.20/16" delay = "50ms" jitter = "50ms" [[device]] node = "node_3" name = "dev4" -cidr = "10.0.0.30/16" +cidr = "11.0.0.30/16" delay = "50ms" jitter = "50ms" [[device]] node = "node_4" name = "dev5" -cidr = "10.0.0.40/16" +cidr = "11.0.0.40/16" delay = "50ms" jitter = "50ms" diff --git a/test-configs/linux/node_0.toml b/test-configs/linux/node_0.toml index 3686c560..76a9dbe6 100644 --- a/test-configs/linux/node_0.toml +++ b/test-configs/linux/node_0.toml @@ -1,12 +1,12 @@ stamp = "/tmp/timeboost.0.stamp" [net.public] -address = "10.0.0.1:8000" -http-api = "10.0.0.1:8004" +address = "11.0.0.1:8000" +http-api = "11.0.0.1:8004" [net.internal] -address = "10.0.0.1:8003" -nitro = "10.0.1.0:55000" +address = "11.0.0.1:8003" +nitro = "11.0.1.0:55000" [keys.signing] secret = "3hzb3bRzn3dXSV1iEVE6mU4BF2aS725s8AboRxLwULPp" @@ -25,8 +25,8 @@ namespace = 10101 [chain.parent] id = 31337 -rpc-url = "http://10.0.1.0:8545/" -ws-url = "ws://10.0.1.0:8545/" +rpc-url = "http://11.0.1.0:8545/" +ws-url = "ws://11.0.1.0:8545/" ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" block-tag = "finalized" key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" diff --git a/test-configs/linux/node_1.toml b/test-configs/linux/node_1.toml index 4f3b1d34..fa3222e8 100644 --- a/test-configs/linux/node_1.toml +++ b/test-configs/linux/node_1.toml @@ -1,12 +1,12 @@ stamp = "/tmp/timeboost.1.stamp" [net.public] -address = "10.0.0.10:8010" -http-api = "10.0.0.10:8014" +address = "11.0.0.10:8010" +http-api = "11.0.0.10:8014" [net.internal] -address = "10.0.0.10:8013" -nitro = "10.0.1.0:55000" +address = "11.0.0.10:8013" +nitro = "11.0.1.0:55000" [keys.signing] secret = "FWJzNGvEjFS3h1N1sSMkcvvroWwjT5LQuGkGHu9JMAYs" @@ -25,8 +25,8 @@ namespace = 10101 [chain.parent] id = 31337 -rpc-url = "http://10.0.1.0:8545/" -ws-url = "ws://10.0.1.0:8545/" +rpc-url = "http://11.0.1.0:8545/" +ws-url = "ws://11.0.1.0:8545/" ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" block-tag = "finalized" key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" diff --git a/test-configs/linux/node_2.toml b/test-configs/linux/node_2.toml index 5f940162..dc9006c2 100644 --- a/test-configs/linux/node_2.toml +++ b/test-configs/linux/node_2.toml @@ -1,12 +1,12 @@ stamp = "/tmp/timeboost.2.stamp" [net.public] -address = "10.0.0.20:8020" -http-api = "10.0.0.20:8024" +address = "11.0.0.20:8020" +http-api = "11.0.0.20:8024" [net.internal] -address = "10.0.0.20:8023" -nitro = "10.0.1.0:55000" +address = "11.0.0.20:8023" +nitro = "11.0.1.0:55000" [keys.signing] secret = "2yWTaC6MWvNva97t81cd9QX5qph68NnB1wRVwAChtuGr" @@ -25,8 +25,8 @@ namespace = 10101 [chain.parent] id = 31337 -rpc-url = "http://10.0.1.0:8545/" -ws-url = "ws://10.0.1.0:8545/" +rpc-url = "http://11.0.1.0:8545/" +ws-url = "ws://11.0.1.0:8545/" ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" block-tag = "finalized" key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" diff --git a/test-configs/linux/node_3.toml b/test-configs/linux/node_3.toml index c116fcd6..020e81e0 100644 --- a/test-configs/linux/node_3.toml +++ b/test-configs/linux/node_3.toml @@ -1,12 +1,12 @@ stamp = "/tmp/timeboost.3.stamp" [net.public] -address = "10.0.0.30:8030" -http-api = "10.0.0.30:8034" +address = "11.0.0.30:8030" +http-api = "11.0.0.30:8034" [net.internal] -address = "10.0.0.30:8033" -nitro = "10.0.1.0:55000" +address = "11.0.0.30:8033" +nitro = "11.0.1.0:55000" [keys.signing] secret = "CUpkbkn8bix7ZrbztPKJwu66MRpJrc1Wr2JfdrhetASk" @@ -25,8 +25,8 @@ namespace = 10101 [chain.parent] id = 31337 -rpc-url = "http://10.0.1.0:8545/" -ws-url = "ws://10.0.1.0:8545/" +rpc-url = "http://11.0.1.0:8545/" +ws-url = "ws://11.0.1.0:8545/" ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" block-tag = "finalized" key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" diff --git a/test-configs/linux/node_4.toml b/test-configs/linux/node_4.toml index e4825577..7998e6fd 100644 --- a/test-configs/linux/node_4.toml +++ b/test-configs/linux/node_4.toml @@ -1,12 +1,12 @@ stamp = "/tmp/timeboost.4.stamp" [net.public] -address = "10.0.0.40:8040" -http-api = "10.0.0.40:8044" +address = "11.0.0.40:8040" +http-api = "11.0.0.40:8044" [net.internal] -address = "10.0.0.40:8043" -nitro = "10.0.1.0:55000" +address = "11.0.0.40:8043" +nitro = "11.0.1.0:55000" [keys.signing] secret = "6LMMEuoPRCkpDsnxnANCRCBC6JagdCHs2pjNicdmQpQE" @@ -25,8 +25,8 @@ namespace = 10101 [chain.parent] id = 31337 -rpc-url = "http://10.0.1.0:8545/" -ws-url = "ws://10.0.1.0:8545/" +rpc-url = "http://11.0.1.0:8545/" +ws-url = "ws://11.0.1.0:8545/" ibox-contract = "0xa0f3a1a4e2b2bcb7b48c8527c28098f207572ec1" block-tag = "finalized" key-manager-contract = "0x2bbf15bc655c4cc157b769cfcb1ea9924b9e1a35" From 1316ba1086814e0328621f9f96ec8b5e9124ca0f Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 19 Sep 2025 21:44:37 +0200 Subject: [PATCH 14/16] Add resolv.conf to namespaces. --- test-utils/src/binaries/net-setup.rs | 45 +++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/test-utils/src/binaries/net-setup.rs b/test-utils/src/binaries/net-setup.rs index 67307842..fe9ff0cb 100644 --- a/test-utils/src/binaries/net-setup.rs +++ b/test-utils/src/binaries/net-setup.rs @@ -46,6 +46,9 @@ fn main() -> Result<()> { if !d.delay.is_zero() { dev.delay(&d.delay, &d.jitter)? } + if c.nat.is_some() { + dev.add_resolv_conf()? + } } if let Some(nat) = c.nat { run_command(TRACE, ["iptables", @@ -62,7 +65,11 @@ fn main() -> Result<()> { let c: Config = toml::from_slice(&t)?; let b = Bridge::new(&c.bridge.name, c.bridge.cidr); for d in c.device { - Device::new(&d).delete()? + let dev = Device::new(&d); + if c.nat.is_some() { + dev.del_resolv_conf()? + } + dev.delete()?; } b.delete()?; if let Some(nat) = c.nat { @@ -113,6 +120,42 @@ impl Device { ]) } + fn add_resolv_conf(&self) -> Result<()> { + const RESOLV_CONF: &str = "nameserver 1.1.1.1\nnameserver 8.8.8.8\n"; + + let dir = PathBuf::from(format!("/etc/netns/{}", self.space)); + if !dir.exists() { + if TRACE { + eprintln!("> creating {dir:?}") + } + fs::create_dir_all(&dir)? + } + let file = dir.join("resolv.conf"); + if TRACE { + eprintln!("> writing {file:?}") + } + fs::write(file, RESOLV_CONF)?; + Ok(()) + } + + fn del_resolv_conf(&self) -> Result<()> { + let dir = PathBuf::from(format!("/etc/netns/{}", self.space)); + if dir.exists() { + let file = dir.join("resolv.conf"); + if file.exists() { + if TRACE { + eprintln!("> removing {file:?}") + } + fs::remove_file(file)? + } + if TRACE { + eprintln!("> removing {dir:?}") + } + fs::remove_dir(dir)? + } + Ok(()) + } + fn delete(self) -> Result<()> { run_command(TRACE, ["ip", "link", "delete", &self.name])?; run_command(TRACE, ["ip", "netns", "delete", &self.space]) From f6d1201d3f747940f8728df0d4da6c2c63d62eab Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Mon, 22 Sep 2025 11:16:19 +0200 Subject: [PATCH 15/16] WIP --- test-utils/src/binaries/net-setup.rs | 36 +++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/test-utils/src/binaries/net-setup.rs b/test-utils/src/binaries/net-setup.rs index fe9ff0cb..70baa817 100644 --- a/test-utils/src/binaries/net-setup.rs +++ b/test-utils/src/binaries/net-setup.rs @@ -57,7 +57,19 @@ fn main() -> Result<()> { "-s", &nat.cidr.to_string(), "-o", &nat.device, "-j", "MASQUERADE" - ])? + ])?; + run_command(TRACE, ["iptables", + "-I", "FORWARD", + "-i", &nat.device, + "-o", "bridge", + "-j", "ACCEPT" + ])?; + run_command(TRACE, ["iptables", + "-I", "FORWARD", + "-i", "bridge", + "-o", &nat.device, + "-j", "ACCEPT" + ])?; } } Command::Delete { config } => { @@ -71,10 +83,28 @@ fn main() -> Result<()> { } dev.delete()?; } - b.delete()?; if let Some(nat) = c.nat { - run_command(TRACE, ["iptables", "-t", &nat.table, "-F"])? + run_command(TRACE, ["iptables", + "-t", &nat.table, + "-D", "POSTROUTING", + "-s", &nat.cidr.to_string(), + "-o", &nat.device, + "-j", "MASQUERADE" + ])?; + run_command(TRACE, ["iptables", + "-D", "FORWARD", + "-i", &nat.device, + "-o", "bridge", + "-j", "ACCEPT" + ])?; + run_command(TRACE, ["iptables", + "-D", "FORWARD", + "-i", "bridge", + "-o", &nat.device, + "-j", "ACCEPT" + ])?; } + b.delete()? } } Ok(()) From c04d3ec4ddeb1266656c66d936519d362f361329 Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Tue, 23 Sep 2025 14:01:08 +0200 Subject: [PATCH 16/16] Add comments. --- Cargo.lock | 2 +- test-configs/linux/net.toml | 1 - test-utils/src/binaries/net-setup.rs | 421 +++++++++++++++------------ test-utils/src/net.rs | 1 - 4 files changed, 240 insertions(+), 185 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe94a9b4..a15ad2f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10909,7 +10909,7 @@ dependencies = [ "timeboost-utils", "tokio", "tokio-util", - "toml 0.9.5", + "toml 0.9.7", "tonic 0.14.2", "tracing", ] diff --git a/test-configs/linux/net.toml b/test-configs/linux/net.toml index 10b9b29c..6eed44d1 100644 --- a/test-configs/linux/net.toml +++ b/test-configs/linux/net.toml @@ -1,5 +1,4 @@ [nat] -table = "nat" device = "eth0" cidr = "11.0.0.0/8" diff --git a/test-utils/src/binaries/net-setup.rs b/test-utils/src/binaries/net-setup.rs index 70baa817..9a415ac7 100644 --- a/test-utils/src/binaries/net-setup.rs +++ b/test-utils/src/binaries/net-setup.rs @@ -1,215 +1,272 @@ -#![cfg_attr(rustfmt, rustfmt_skip)] - -use std::{fs, path::PathBuf}; - -use anyhow::{Result, ensure}; -use clap::Parser; -use ipnet::Ipv4Net; -use jiff::Span; -use test_utils::{ - net::{Config, DeviceConfig}, - process::run_command, -}; - -const TRACE: bool = true; - -#[derive(Debug, Parser)] -enum Command { - System { - #[clap(long, action = clap::ArgAction::Set)] - forward_ipv4: bool, - }, - Create { - #[clap(long, short)] - config: PathBuf, - }, - Delete { - #[clap(long, short)] - config: PathBuf, - }, +//! This executable configures network namespaces according to a configuration file. +//! +//! It creates a bridge device, *N* namespaces and virtual ethernet device pairs, +//! with one end inside the namespace. It also applies a queuing discipline to +//! the egress traffic of each device using the configured delay and jitter +//! values. Finally, for each namespace a `resolv.conf` file is created in +//! `/etc/netns//` to ensure name resolution works inside the namespace. +//! +//! Here is a sample trace showing the command execution for the bridge device and +//! one namespace plus device, and finally the iptables setup. +//! +//! ``` +//! > ip link add bridge type bridge +//! > ip addr add 11.0.1.0/16 dev bridge +//! > ip link set bridge up +//! > ip netns add ns-dev1 +//! > ip link add dev-dev1 type veth peer name dev1 +//! > ip link set dev1 up +//! > ip link set dev1 master bridge +//! > ip link set dev-dev1 netns ns-dev1 +//! > ip netns exec ns-dev1 ip addr add 11.0.0.1/16 dev dev-dev1 +//! > ip netns exec ns-dev1 ip link set dev-dev1 up +//! > ip netns exec ns-dev1 ip link set lo up +//! > ip netns exec ns-dev1 ip route add default via 11.0.1.0 +//! > ip netns exec ns-dev1 tc qdisc add dev dev-dev1 root netem delay 50ms 50ms +//! > writing "/etc/netns/ns-dev1/resolv.conf" +//! ... +//! > iptables -t nat -A POSTROUTING -s 11.0.0.0/8 -o eth0 -j MASQUERADE +//! > iptables -I FORWARD -i eth0 -o bridge -j ACCEPT +//! > iptables -I FORWARD -i bridge -o eth0 -j ACCEPT +//! ``` +//! +//! Should the setup encounter an error, manual cleanup may be required: +//! +//! 1. To remove any `resolv.conf` files, remove the sub-directories in `/etc/netns` +//! 2. To remove the bridge device, enter (as root): `ip link delete bridge`. +//! 3. To remove a virtual ethernet device, enter (as root): `ip link delete ` +//! 4. To remove a namespace, enter (as root): `ip netns delete `. +//! 5. To remove iptables settings, enter (as root): +//! - `iptables -t nat -D POSTROUTING -s -o -j MASQUERADE` +//! - `iptables -D FORWARD -i -o bridge -j ACCEPT` +//! - `iptables -D FORWARD -o -i bridge -j ACCEPT` + +use anyhow::Result; + +#[cfg(not(target_os = "linux"))] +fn main() -> Result<()> { + Ok(()) } +#[cfg(target_os = "linux")] fn main() -> Result<()> { - match Command::parse() { - Command::System { forward_ipv4 } => { - let v = u8::from(forward_ipv4); - run_command(TRACE, ["sysctl", "-w", &format!("net.ipv4.ip_forward={v}")])? - } - Command::Create { config } => { - let t = fs::read(&config)?; - let c: Config = toml::from_slice(&t)?; - let b = Bridge::new(&c.bridge.name, c.bridge.cidr); - b.create()?; - for d in c.device { - let dev = Device::new(&d); - dev.create(&b)?; - if !d.delay.is_zero() { - dev.delay(&d.delay, &d.jitter)? + setup::go() +} + +#[cfg(target_os = "linux")] +#[rustfmt::skip] +mod setup { + use std::{fs, path::PathBuf}; + + use anyhow::{Result, ensure}; + use clap::Parser; + use ipnet::Ipv4Net; + use jiff::Span; + use test_utils::{ + net::{Config, DeviceConfig}, + process::run_command, + }; + + const TRACE: bool = true; + + pub fn go() -> Result<()> { + match Command::parse() { + Command::System { forward_ipv4 } => { + let v = u8::from(forward_ipv4); + run_command(TRACE, ["sysctl", "-w", &format!("net.ipv4.ip_forward={v}")])? + } + Command::Create { config } => { + let t = fs::read(&config)?; + let c: Config = toml::from_slice(&t)?; + let b = Bridge::new(&c.bridge.name, c.bridge.cidr); + b.create()?; + for d in c.device { + let dev = Device::new(&d); + dev.create(&b)?; + if !d.delay.is_zero() { + dev.delay(&d.delay, &d.jitter)? + } + if c.nat.is_some() { + dev.add_resolv_conf()? + } } - if c.nat.is_some() { - dev.add_resolv_conf()? + if let Some(nat) = c.nat { + run_command(TRACE, ["iptables", + "-t", "nat", + "-A", "POSTROUTING", + "-s", &nat.cidr.to_string(), + "-o", &nat.device, + "-j", "MASQUERADE" + ])?; + run_command(TRACE, ["iptables", + "-I", "FORWARD", + "-i", &nat.device, + "-o", "bridge", + "-j", "ACCEPT" + ])?; + run_command(TRACE, ["iptables", + "-I", "FORWARD", + "-i", "bridge", + "-o", &nat.device, + "-j", "ACCEPT" + ])?; } } - if let Some(nat) = c.nat { - run_command(TRACE, ["iptables", - "-t", &nat.table, - "-A", "POSTROUTING", - "-s", &nat.cidr.to_string(), - "-o", &nat.device, - "-j", "MASQUERADE" - ])?; - run_command(TRACE, ["iptables", - "-I", "FORWARD", - "-i", &nat.device, - "-o", "bridge", - "-j", "ACCEPT" - ])?; - run_command(TRACE, ["iptables", - "-I", "FORWARD", - "-i", "bridge", - "-o", &nat.device, - "-j", "ACCEPT" - ])?; - } - } - Command::Delete { config } => { - let t = fs::read(&config)?; - let c: Config = toml::from_slice(&t)?; - let b = Bridge::new(&c.bridge.name, c.bridge.cidr); - for d in c.device { - let dev = Device::new(&d); - if c.nat.is_some() { - dev.del_resolv_conf()? + Command::Delete { config } => { + let t = fs::read(&config)?; + let c: Config = toml::from_slice(&t)?; + let b = Bridge::new(&c.bridge.name, c.bridge.cidr); + for d in c.device { + let dev = Device::new(&d); + if c.nat.is_some() { + dev.del_resolv_conf()? + } + dev.delete()?; } - dev.delete()?; - } - if let Some(nat) = c.nat { - run_command(TRACE, ["iptables", - "-t", &nat.table, - "-D", "POSTROUTING", - "-s", &nat.cidr.to_string(), - "-o", &nat.device, - "-j", "MASQUERADE" - ])?; - run_command(TRACE, ["iptables", - "-D", "FORWARD", - "-i", &nat.device, - "-o", "bridge", - "-j", "ACCEPT" - ])?; - run_command(TRACE, ["iptables", - "-D", "FORWARD", - "-i", "bridge", - "-o", &nat.device, - "-j", "ACCEPT" - ])?; + if let Some(nat) = c.nat { + run_command(TRACE, ["iptables", + "-t", "nat", + "-D", "POSTROUTING", + "-s", &nat.cidr.to_string(), + "-o", &nat.device, + "-j", "MASQUERADE" + ])?; + run_command(TRACE, ["iptables", + "-D", "FORWARD", + "-i", &nat.device, + "-o", "bridge", + "-j", "ACCEPT" + ])?; + run_command(TRACE, ["iptables", + "-D", "FORWARD", + "-i", "bridge", + "-o", &nat.device, + "-j", "ACCEPT" + ])?; + } + b.delete()? } - b.delete()? - } - } - Ok(()) -} - -#[derive(Debug)] -struct Device { - space: String, - name: String, - dev: String, - cidr: Ipv4Net, -} - -impl Device { - fn new(cfg: &DeviceConfig) -> Device { - Self { - space: cfg.namespace(), - dev: cfg.device(), - cidr: cfg.cidr, - name: cfg.name.clone(), } + Ok(()) } - fn create(&self, b: &Bridge) -> Result<()> { - ensure!(b.net.contains(&self.cidr)); - run_command(TRACE, ["ip", "netns", "add", &self.space])?; - run_command(TRACE, ["ip", "link", "add", &self.dev, "type", "veth", "peer", "name", &self.name])?; - run_command(TRACE, ["ip", "link", "set", &self.name, "up"])?; - run_command(TRACE, ["ip", "link", "set", &self.name, "master", &b.name])?; - run_command(TRACE, ["ip", "link", "set", &self.dev, "netns", &self.space])?; - run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "addr", "add", &self.cidr.to_string(), "dev", &self.dev])?; - run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "link", "set", &self.dev, "up"])?; - run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "link", "set", "lo", "up"])?; - run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "route", "add", "default", "via", &b.net.addr().to_string()]) + #[derive(Debug, Parser)] + enum Command { + System { + #[clap(long, action = clap::ArgAction::Set)] + forward_ipv4: bool, + }, + Create { + #[clap(long, short)] + config: PathBuf, + }, + Delete { + #[clap(long, short)] + config: PathBuf, + }, } - fn delay(&self, delay: &Span, jitter: &Span) -> Result<()> { - let d = format!("{}ms", delay.get_milliseconds()); - let j = format!("{}ms", jitter.get_milliseconds()); - run_command(TRACE, [ - "ip", "netns", "exec", &self.space, - "tc", "qdisc", "add", "dev", &self.dev, "root", "netem", "delay", &d, &j - ]) + #[derive(Debug)] + struct Device { + space: String, + name: String, + dev: String, + cidr: Ipv4Net, } - fn add_resolv_conf(&self) -> Result<()> { - const RESOLV_CONF: &str = "nameserver 1.1.1.1\nnameserver 8.8.8.8\n"; - - let dir = PathBuf::from(format!("/etc/netns/{}", self.space)); - if !dir.exists() { - if TRACE { - eprintln!("> creating {dir:?}") + impl Device { + fn new(cfg: &DeviceConfig) -> Self { + Self { + space: cfg.namespace(), + dev: cfg.device(), + cidr: cfg.cidr, + name: cfg.name.clone(), } - fs::create_dir_all(&dir)? } - let file = dir.join("resolv.conf"); - if TRACE { - eprintln!("> writing {file:?}") + + fn create(&self, b: &Bridge) -> Result<()> { + ensure!(b.net.contains(&self.cidr)); + run_command(TRACE, ["ip", "netns", "add", &self.space])?; + run_command(TRACE, ["ip", "link", "add", &self.dev, "type", "veth", "peer", "name", &self.name])?; + run_command(TRACE, ["ip", "link", "set", &self.name, "up"])?; + run_command(TRACE, ["ip", "link", "set", &self.name, "master", &b.name])?; + run_command(TRACE, ["ip", "link", "set", &self.dev, "netns", &self.space])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "addr", "add", &self.cidr.to_string(), "dev", &self.dev])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "link", "set", &self.dev, "up"])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "link", "set", "lo", "up"])?; + run_command(TRACE, ["ip", "netns", "exec", &self.space, "ip", "route", "add", "default", "via", &b.net.addr().to_string()]) } - fs::write(file, RESOLV_CONF)?; - Ok(()) - } - fn del_resolv_conf(&self) -> Result<()> { - let dir = PathBuf::from(format!("/etc/netns/{}", self.space)); - if dir.exists() { - let file = dir.join("resolv.conf"); - if file.exists() { + fn delay(&self, delay: &Span, jitter: &Span) -> Result<()> { + let d = format!("{}ms", delay.get_milliseconds()); + let j = format!("{}ms", jitter.get_milliseconds()); + run_command(TRACE, [ + "ip", "netns", "exec", &self.space, + "tc", "qdisc", "add", "dev", &self.dev, "root", "netem", "delay", &d, &j + ]) + } + + fn add_resolv_conf(&self) -> Result<()> { + const RESOLV_CONF: &str = "nameserver 1.1.1.1\nnameserver 8.8.8.8\n"; + + let dir = PathBuf::from(format!("/etc/netns/{}", self.space)); + if !dir.exists() { if TRACE { - eprintln!("> removing {file:?}") + eprintln!("> creating {dir:?}") } - fs::remove_file(file)? + fs::create_dir_all(&dir)? } + let file = dir.join("resolv.conf"); if TRACE { - eprintln!("> removing {dir:?}") + eprintln!("> writing {file:?}") } - fs::remove_dir(dir)? + fs::write(file, RESOLV_CONF)?; + Ok(()) } - Ok(()) - } - - fn delete(self) -> Result<()> { - run_command(TRACE, ["ip", "link", "delete", &self.name])?; - run_command(TRACE, ["ip", "netns", "delete", &self.space]) - } -} -#[derive(Debug)] -struct Bridge { - name: String, - net: Ipv4Net, -} + fn del_resolv_conf(&self) -> Result<()> { + let dir = PathBuf::from(format!("/etc/netns/{}", self.space)); + if dir.exists() { + let file = dir.join("resolv.conf"); + if file.exists() { + if TRACE { + eprintln!("> removing {file:?}") + } + fs::remove_file(file)? + } + if TRACE { + eprintln!("> removing {dir:?}") + } + fs::remove_dir(dir)? + } + Ok(()) + } -impl Bridge { - fn new(name: &str, ip: Ipv4Net) -> Self { - Self { name: name.to_string(), net: ip } + fn delete(self) -> Result<()> { + run_command(TRACE, ["ip", "link", "delete", &self.name])?; + run_command(TRACE, ["ip", "netns", "delete", &self.space]) + } } - fn create(&self) -> Result<()> { - run_command(TRACE, ["ip", "link", "add", &self.name, "type", "bridge"])?; - run_command(TRACE, ["ip", "addr", "add", &self.net.to_string(), "dev", "bridge"])?; - run_command(TRACE, ["ip", "link", "set", &self.name, "up"]) + #[derive(Debug)] + struct Bridge { + name: String, + net: Ipv4Net, } - fn delete(self) -> Result<()> { - run_command(TRACE, ["ip", "link", "delete", &self.name]) + impl Bridge { + fn new(name: &str, ip: Ipv4Net) -> Self { + Self { name: name.to_string(), net: ip } + } + + fn create(&self) -> Result<()> { + run_command(TRACE, ["ip", "link", "add", &self.name, "type", "bridge"])?; + run_command(TRACE, ["ip", "addr", "add", &self.net.to_string(), "dev", "bridge"])?; + run_command(TRACE, ["ip", "link", "set", &self.name, "up"]) + } + + fn delete(self) -> Result<()> { + run_command(TRACE, ["ip", "link", "delete", &self.name]) + } } } diff --git a/test-utils/src/net.rs b/test-utils/src/net.rs index a2738a3c..279b5507 100644 --- a/test-utils/src/net.rs +++ b/test-utils/src/net.rs @@ -30,7 +30,6 @@ pub struct DeviceConfig { #[derive(Deserialize)] pub struct NatConfig { - pub table: String, pub device: String, pub cidr: Ipv4Net, }