diff --git a/.gitignore b/.gitignore index 7cef681d89..d5f479133b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ cargo-test-* tarpaulin-report.html + +perf/docker/work/ + diff --git a/perf/docker/Dockerfile b/perf/docker/Dockerfile new file mode 100644 index 0000000000..d7679d8eda --- /dev/null +++ b/perf/docker/Dockerfile @@ -0,0 +1,14 @@ +# https://hub.docker.com/_/debian +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y procps iputils-ping iproute2 ethtool tcpdump + +WORKDIR /root + +RUN mkdir -p .local/share/quinn + +COPY ./entrypoint.sh . + +COPY ./target/quinn-perf . + +ENTRYPOINT [ "/root/entrypoint.sh" ] diff --git a/perf/docker/README.md b/perf/docker/README.md new file mode 100644 index 0000000000..f38c544c7c --- /dev/null +++ b/perf/docker/README.md @@ -0,0 +1,119 @@ +# Quinn performance network simulator + +This docker file and set of scripts allow running a performance test using +`quinn-perf` binary. + +## Building + +### Standard binary + +To build the `quinn-perf` binary and docker image use the following command: + +```sh +./build.sh +``` + +### Instrumented binary + +To build an instrumented `quinn-perf` binary that allows supervision using +[`tokio-console`](https://github.com/tokio-rs/console), use the following +command: + +```sh +./build.sh -t +``` + +## Launching + +### Server + +To launch `quinn-perf server` use the following command: +```sh +./run-server.sh +``` + +### Client + +To launch `quinn-perf client` use the following command: +```sh +./run-client.sh +``` +It connects to `quinn-perf server` running in docker. + +### Arguments + +| name | help | description | +| :-: | :-: | :-: | +| `-h` | display help | | +| `-c` | enable packet capture | capture will be stored in `./work/` | +| `-g` | enabled GSO | default: disabled | +| `-l num` | specify simulated latency in ms | default: 0ms | +| `-L num` | specify simulated packet loss in percentage | default: 0% | +| `-o` | open packet capture | requires `-c` | +| `-t` | attach to tokio console | requires an instrumented binary | + +#### Change default `quinn-perf` arguments + +`--` can be used to add additional arguments for `quinn-perf` binary. + +Example: +```sh +./run-client.sh -g -l 5 -- --help +``` +shows `quinn-perf` available arguments. + +Example: +```sh +./run-client.sh -g -l 5 -- --download-size 5M --upload-size 5M --duration 10 +``` +runs a bidirectional download and upload benchmark of 5 MB per stream for 10 +seconds. + +## Simulate network latency or packet loss + +Argument `-l` can be used to simulate network latency. + +If you want to simulate a 10ms latency, launch server with `-l 5` argument and +launch client with `-l 5` argument. + +Argument `-L` can be used to simulate network packet loss. + +If you want to simulate a `0.1%` loss link, launch server with `-L 0.1` argument and +launch client with standard arguments. + +Latency is simulated at network interface level using linux kernel QoS ([`tc +netem`](https://man7.org/linux/man-pages/man8/tc-netem.8.html)). + +Scripts display some statistics at the end of the run. + +## Capture packets + +Argument `-c` can be used to enable packet capture. Dumps are stored in +`./work/`. By default `quinn-perf` is configured to dump the cryptographic keys +in the same folder. + +Argument `-o` can be used to open the packet capture at the end of the +benchmark using [`wireshark`](https://www.wireshark.org/) (which must be +installed). + +## Analyze using `tokio-console` + +To instrument and monitor `quinn-perf` using `tokio-console` (which must be installed), compile using: +```sh +./build.sh -t +``` + +Then launch server and client using wanted options. + +Then run: +```sh +./run-server.sh -t +``` +to launch `tokio-console` and attach it to server binary. + +Then run: +```sh +./run-client.sh -t +``` +to launch `tokio-console` and attach it to client binary. + diff --git a/perf/docker/build.sh b/perf/docker/build.sh new file mode 100755 index 0000000000..4912cf4f45 --- /dev/null +++ b/perf/docker/build.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env -S bash -eu + +CURDIR=$(pwd) +TOKIO_CONSOLE=0 + +function usage() { + echo "usage: $0 [-t]" + echo " -t enable tokio console" + exit 1 +} + +while getopts "t" opt; do + case $opt in + t) TOKIO_CONSOLE=1;; + h) usage;; + *) usage;; + esac +done + +mkdir -p ./target + +cd ../.. + +if [ ${TOKIO_CONSOLE} -eq 0 ]; then + cargo build -p perf --release +else + echo "Building with tokio console support" + RUSTFLAGS="--cfg tokio_unstable" cargo build -p perf -r -F tokio-console +fi + + +cp -au ./target/release/quinn-perf ${CURDIR}/target + +cd ${CURDIR} +docker compose build diff --git a/perf/docker/docker-compose.yml b/perf/docker/docker-compose.yml new file mode 100644 index 0000000000..7b50efef45 --- /dev/null +++ b/perf/docker/docker-compose.yml @@ -0,0 +1,38 @@ +networks: + quinn-perf: + ipam: + config: + - subnet: 172.42.0.0/16 + +services: + server: + build: . + image: quinn_perf + networks: + quinn-perf: + ipv4_address: 172.42.0.2 + volumes: + - ./work:/root/.local/share/quinn + cap_add: + - NET_ADMIN + environment: + - SSLKEYLOGFILE=/root/.local/share/quinn/server.key + - TOKIO_CONSOLE_BIND=0.0.0.0:6669 # tokio-console + ports: + - 6669:6669 # tokio-console + + client: + image: quinn_perf + networks: + quinn-perf: + ipv4_address: 172.42.0.3 + volumes: + - ./work:/root/.local/share/quinn + cap_add: + - NET_ADMIN + environment: + - SSLKEYLOGFILE=/root/.local/share/quinn/client.key + - TOKIO_CONSOLE_BIND=0.0.0.0:6669 # tokio-console + ports: + - 6668:6669 # tokio-console + diff --git a/perf/docker/entrypoint.sh b/perf/docker/entrypoint.sh new file mode 100755 index 0000000000..423c24e5e0 --- /dev/null +++ b/perf/docker/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +while :; do sleep 10; done diff --git a/perf/docker/run-client.sh b/perf/docker/run-client.sh new file mode 100755 index 0000000000..6000e7650f --- /dev/null +++ b/perf/docker/run-client.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env -S bash -eu + +SERVICE=client + +# quinn-perf client arguments +QUINN_PERF_SERVER="172.42.0.2:4433" +QUINN_PERF_DEFAULT_ARGS="--keylog" + +LATENCY=0 +LOSS=0 +GSO=0 +CAPTURE=0 +OPEN=0 +TOKIO_CONSOLE=0 + +function usage() { + echo "usage: $0 [-cghot] [-l number] [-L number]" + echo " -c enable packet capture" + echo " -g enable GSO (default: disabled)" + echo " -h display help" + echo " -l number specify simulated latency in ms (default: ${LATENCY}ms)" + echo " -L number specify simulated packet loss in percentage (default: ${LOSS}%)" + echo " -o open packet capture" + echo " -t attach to tokio console" + echo " -- args can be used to add extra arguments to quinn-perf" + exit 1 +} + +while getopts "cghl:L:ot" opt; do + case $opt in + c) CAPTURE=1;; + g) GSO=1;; + l) LATENCY=$OPTARG;; + L) LOSS=$OPTARG;; + o) OPEN=1;; + t) TOKIO_CONSOLE=1;; + h) usage;; + *) usage;; + esac +done + +# extract optional additional arguments for quinn-perf +shift "$((OPTIND - 1))" +QUINN_PERF_EXTRA_ARGS="$*" + +if [ ${TOKIO_CONSOLE} -eq 1 ]; then + tokio-console http://127.0.0.1:6668 + exit 0 +fi + +mkdir -p ./work + +echo "Launching docker ${SERVICE}" +docker compose up -d --force-recreate ${SERVICE} +if [ "${LATENCY}" -ne "0" ] || [ "${LOSS}" -ne "0" ]; then + echo "Enforcing a latency of ${LATENCY}ms and a packet loss of ${LOSS}%" + docker compose exec -it ${SERVICE} tc qdisc add dev eth0 root netem delay "${LATENCY}ms" loss "${LOSS}%" +fi + +if [ ${GSO} -eq 0 ]; then + # FIXME disable GSO due to this issue + # https://gitlab.com/wireshark/wireshark/-/issues/19109 + docker compose exec -it ${SERVICE} ethtool -K eth0 tx-udp-segmentation off +fi + +if [ ${CAPTURE} -eq 1 ]; then + echo "Starting capture within docker" + docker compose exec -d ${SERVICE} tcpdump -ni eth0 -s0 -w /root/.local/share/quinn/${SERVICE}.pcap udp and port 4433 +fi + +echo "Launching quinn-perf client with arguments: ${QUINN_PERF_DEFAULT_ARGS} ${QUINN_PERF_EXTRA_ARGS} ${QUINN_PERF_SERVER}" +docker compose exec -it ${SERVICE} /root/quinn-perf client ${QUINN_PERF_DEFAULT_ARGS} ${QUINN_PERF_EXTRA_ARGS} ${QUINN_PERF_SERVER} + +if [ ${CAPTURE} -eq 1 ]; then + echo "Stopping capture within docker" + docker compose exec -it ${SERVICE} killall -STOP tcpdump +fi + +if [ "${LATENCY}" -ne "0" ] || [ "${LOSS}" -ne "0" ]; then + echo "Dumping QOS stats" + docker compose exec -it ${SERVICE} tc -s qdisc ls dev eth0 +fi + +if [ ${CAPTURE} -eq 1 ] && [ ${OPEN} -eq 1 ]; then + wireshark -o tls.keylog_file:./work/${SERVICE}.key ./work/${SERVICE}.pcap +fi diff --git a/perf/docker/run-server.sh b/perf/docker/run-server.sh new file mode 100755 index 0000000000..1893f9ac9f --- /dev/null +++ b/perf/docker/run-server.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env -S bash -eu + +SERVICE=server + +# quinn-perf server arguments +QUINN_PERF_DEFAULT_ARGS="--keylog --listen 172.42.0.2:4433" + +LATENCY=0 +LOSS=0 +GSO=0 +CAPTURE=0 +OPEN=0 +TOKIO_CONSOLE=0 + +function usage() { + echo "usage: $0 [-cghot] [-l number] [-L number]" + echo " -c enable packet capture" + echo " -g enable GSO (default: disabled)" + echo " -h display help" + echo " -l number specify simulated latency in ms (default: ${LATENCY}ms)" + echo " -L number specify simulated packet loss in percentage (default: ${LOSS}%)" + echo " -o open packet capture" + echo " -t attach to tokio console" + echo " -- args can be used to add extra arguments to quinn-perf" + exit 1 +} + +while getopts "cghl:L:ot" opt; do + case $opt in + c) CAPTURE=1;; + g) GSO=1;; + l) LATENCY=$OPTARG;; + L) LOSS=$OPTARG;; + o) OPEN=1;; + t) TOKIO_CONSOLE=1;; + h) usage;; + *) usage;; + esac +done + +# extract optional additional arguments for quinn-perf +shift "$((OPTIND - 1))" +QUINN_PERF_EXTRA_ARGS="$*" + +if [ ${TOKIO_CONSOLE} -eq 1 ]; then + tokio-console http://127.0.0.1:6669 + exit 0 +fi + +mkdir -p ./work + +echo "Launching docker ${SERVICE}" +docker compose up -d --force-recreate ${SERVICE} +if [ "${LATENCY}" -ne "0" ] || [ "${LOSS}" -ne "0" ]; then + echo "Enforcing a latency of ${LATENCY}ms and a packet loss of ${LOSS}%" + docker compose exec -it ${SERVICE} tc qdisc add dev eth0 root netem delay "${LATENCY}ms" loss "${LOSS}%" +fi + +if [ ${GSO} -eq 0 ]; then + # FIXME disable GSO due to this issue + # https://gitlab.com/wireshark/wireshark/-/issues/19109 + docker compose exec -it ${SERVICE} ethtool -K eth0 tx-udp-segmentation off +fi + +if [ ${CAPTURE} -eq 1 ]; then + echo "Starting capture within docker" + docker compose exec -d ${SERVICE} tcpdump -ni eth0 -s0 -w /root/.local/share/quinn/${SERVICE}.pcap udp port 4433 +fi + +echo "Launching quinn perf server with arguments: ${QUINN_PERF_DEFAULT_ARGS} ${QUINN_PERF_EXTRA_ARGS}" +docker compose exec -d ${SERVICE} /root/quinn-perf server ${QUINN_PERF_DEFAULT_ARGS} ${QUINN_PERF_EXTRA_ARGS} + +echo "Press Ctrl-C to stop server" +( trap exit SIGINT ; read -r -d '' _