Skip to content

Commit 81adbb3

Browse files
zhangchuang01sequix
zhangchuang01
authored and
sequix
committed
Filesystem benchmark based on fio.
To make a contrast, this benchmark will test all three types of image, OCI, stargz and estargz in following steps. 1.Use fio to generate fake pread() requests parallely (4 threads). 2.Scrape metrics in /proc/<pid of stargz-snapshotter>. 3.Calculate scraped metrics with PromQL. 4.Draw fio bandwidth-latency and process-metrics images with gunplot. Signed-off-by: Chuang Zhang <sequix@163.com>
1 parent fff955f commit 81adbb3

File tree

16 files changed

+1236
-2
lines changed

16 files changed

+1236
-2
lines changed

‎Makefile‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ GO111MODULE_VALUE=auto
1919
PREFIX ?= out/
2020

2121
CMD=containerd-stargz-grpc ctr-remote
22-
2322
CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))
23+
BENCHMARK_OUTPUT=out-bench/
2424

2525
.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-pullsecrets test-cri
2626

@@ -61,6 +61,7 @@ uninstall:
6161
clean:
6262
@echo "$@"
6363
@rm -f $(CMD_BINARIES)
64+
@rm -rf $(BENCHMARK_OUTPUT)
6465

6566
test:
6667
@echo "$@"
@@ -81,6 +82,9 @@ test-optimize:
8182
benchmark:
8283
@./script/benchmark/test.sh
8384

85+
fs-bench:
86+
@./script/fs-bench/test.sh
87+
8488
test-pullsecrets:
8589
@./script/pullsecrets/test.sh
8690

‎script/fs-bench/fio/Dockerfile‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Copyright The containerd Authors.
2+
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM alpine:3.9 AS stage-build
16+
17+
WORKDIR /work
18+
RUN apk add git build-base linux-headers && \
19+
git clone git://git.kernel.dk/fio.git /work/fio && \
20+
cd /work/fio && \
21+
git checkout c96b385b6e0c78478697713e6da9174fba2432d3 && \
22+
./configure --prefix=/work/root && \
23+
make && \
24+
make install
25+
26+
FROM alpine:3.9 AS stage-final
27+
28+
# How many bytes fio is going to read/write, 512MiB by default.
29+
ARG size=256M
30+
31+
# How many concurrent reading threads.
32+
ARG thread=4
33+
34+
WORKDIR /work
35+
COPY --from=stage-build /work/root/ /
36+
37+
# Lay out test files here, so that they can be compressed into the image layer.
38+
RUN fio -directory=/work -direct=1 -numjobs=$thread -size=$size -name=test -rw=read

‎script/fs-bench/test.sh‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
3+
# Copyright The containerd Authors.
4+
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
19+
cleanup() {
20+
local ORG_EXIT_CODE="${1}"
21+
rm "${DOCKER_COMPOSE_YAML}" || true
22+
exit "${ORG_EXIT_CODE}"
23+
}
24+
trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM
25+
26+
CONTEXT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/"
27+
REPO="${CONTEXT}../../"
28+
29+
echo "Preparing docker-compose.yml..."
30+
DOCKER_COMPOSE_YAML=$(mktemp)
31+
BENCHMARKING_NODE=fs-bench
32+
BENCHMARKING_CONTAINER=fs-bench
33+
cat <<EOF > "${DOCKER_COMPOSE_YAML}"
34+
version: "3"
35+
services:
36+
${BENCHMARKING_NODE}:
37+
build:
38+
context: "${CONTEXT}/work"
39+
dockerfile: Dockerfile
40+
container_name: ${BENCHMARKING_CONTAINER}
41+
privileged: true
42+
working_dir: /go/src/github.com/containerd/stargz-snapshotter
43+
command: tail -f /dev/null
44+
environment:
45+
- NO_PROXY=127.0.0.1,localhost
46+
- HTTP_PROXY=${HTTP_PROXY:-}
47+
- HTTPS_PROXY=${HTTPS_PROXY:-}
48+
- http_proxy=${http_proxy:-}
49+
- https_proxy=${https_proxy:-}
50+
tmpfs:
51+
- /tmp:exec,mode=777
52+
volumes:
53+
- "${REPO}:/go/src/github.com/containerd/stargz-snapshotter:ro"
54+
- "/dev/fuse:/dev/fuse"
55+
- "containerd-data:/var/lib/containerd:delegated"
56+
- "containerd-stargz-grpc-data:/var/lib/containerd-stargz-grpc:delegated"
57+
- "containerd-stargz-grpc-status:/run/containerd-stargz-grpc:delegated"
58+
volumes:
59+
containerd-data:
60+
containerd-stargz-grpc-data:
61+
containerd-stargz-grpc-status:
62+
EOF
63+
64+
echo "Setup benchmark environment..."
65+
docker-compose -f "${DOCKER_COMPOSE_YAML}" build ${DOCKER_BUILD_ARGS:-} "${BENCHMARKING_NODE}"
66+
docker-compose -f "${DOCKER_COMPOSE_YAML}" up -d --force-recreate
67+
68+
echo "Benchmarking..."
69+
docker exec -i "${BENCHMARKING_CONTAINER}" script/fs-bench/work/run.sh
70+
71+
echo "Harvesting output..."
72+
docker cp "${BENCHMARKING_CONTAINER}:/output" "${REPO}/out-bench"
73+
74+
echo "Cleaning up benchmark environment..."
75+
docker-compose -f "${DOCKER_COMPOSE_YAML}" down -v

‎script/fs-bench/work/Dockerfile‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright The containerd Authors.
2+
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM golang:1.13
16+
17+
# basic tools
18+
COPY ./tools /tmp/tools
19+
20+
RUN apt-get update -y && \
21+
apt-get install --no-install-recommends -y libbtrfs-dev libseccomp-dev fuse python \
22+
apt-transport-https software-properties-common && \
23+
curl -Lo gnuplot-5.2.8.tar.gz \
24+
https://sourceforge.net/projects/gnuplot/files/gnuplot/5.2.8/gnuplot-5.2.8.tar.gz/download && \
25+
tar xf gnuplot-5.2.8.tar.gz && \
26+
cd gnuplot-5.2.8 && \
27+
./configure && make && make install && \
28+
cd /tmp/tools && \
29+
GO111MODULE=on go build -o "/usr/local/bin/process" "./process/main.go" && \
30+
GO111MODULE=on go build -o "/usr/local/bin/scrape" "./scrape/main.go"
31+
32+
# runtime dependencies
33+
RUN git clone https://github.com/opencontainers/runc \
34+
$GOPATH/src/github.com/opencontainers/runc && \
35+
cd $GOPATH/src/github.com/opencontainers/runc && \
36+
git checkout d736ef14f0288d6993a1845745d6756cfc9ddd5a && \
37+
GO111MODULE=off make -j2 BUILDTAGS='seccomp apparmor' && \
38+
GO111MODULE=off make install && \
39+
git clone https://github.com/containerd/containerd \
40+
$GOPATH/src/github.com/containerd/containerd && \
41+
cd $GOPATH/src/github.com/containerd/containerd && \
42+
git checkout 990076b731ec9446437972b41176a6b0f3b7bcbf && \
43+
GO111MODULE=off make -j2 && GO111MODULE=off make install

‎script/fs-bench/work/config/config.containerd.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[proxy_plugins]
2+
[proxy_plugins.stargz]
3+
type = "snapshot"
4+
address = "/run/containerd-stargz-grpc/containerd-stargz-grpc.sock"

‎script/fs-bench/work/config/config.stargz.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
noprefetch = true

‎script/fs-bench/work/config/fio.conf‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[test]
2+
# TODO randomize the size of each read (pread, to be precise).
3+
bs=4k
4+
rw=randread
5+
size=256M
6+
numjobs=4
7+
directory=/work
8+
write_bw_log=/output/test
9+
write_iops_log=/output/test
10+
write_lat_log=/output/test

‎script/fs-bench/work/reset.sh‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/bash
2+
3+
# Copyright The containerd Authors.
4+
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
19+
CONTAINERD_ROOT=/var/lib/containerd/
20+
CONTAINERD_CONFIG_DIR=/etc/containerd/
21+
REMOTE_SNAPSHOTTER_SOCKET=/run/containerd-stargz-grpc/containerd-stargz-grpc.sock
22+
REMOTE_SNAPSHOTTER_ROOT=/var/lib/containerd-stargz-grpc/
23+
REMOTE_SNAPSHOTTER_CONFIG_DIR=/etc/containerd-stargz-grpc/
24+
25+
for arg; do
26+
case x"$arg" in
27+
x-nosnapshotter)
28+
NOSNAPSHOTTER="-nosnapshotter"
29+
;;
30+
x-nocleanup)
31+
NOCLEANUP="-nocleanup"
32+
;;
33+
x*)
34+
SCRAPE_OUTPUT_FILENAME="$arg"
35+
;;
36+
esac
37+
done
38+
39+
RETRYNUM=30
40+
RETRYINTERVAL=1
41+
TIMEOUTSEC=180
42+
function retry {
43+
local SUCCESS=false
44+
for i in $(seq ${RETRYNUM}) ; do
45+
if eval "timeout ${TIMEOUTSEC} ${@}" ; then
46+
SUCCESS=true
47+
break
48+
fi
49+
echo "Fail(${i}). Retrying..."
50+
sleep ${RETRYINTERVAL}
51+
done
52+
if [ "${SUCCESS}" == "true" ] ; then
53+
return 0
54+
else
55+
return 1
56+
fi
57+
}
58+
59+
function kill_all {
60+
if [ "${1}" != "" ] ; then
61+
ps aux | grep "${1}" | grep -v grep | grep -v $(basename ${0}) | sed -E 's/ +/ /g' | cut -f 2 -d ' ' | xargs -I{} kill -9 {} || true
62+
fi
63+
}
64+
65+
function cleanup {
66+
rm -rf "${CONTAINERD_ROOT}"*
67+
if [ -f "${REMOTE_SNAPSHOTTER_SOCKET}" ] ; then
68+
rm "${REMOTE_SNAPSHOTTER_SOCKET}"
69+
fi
70+
if [ -d "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" ] ; then
71+
find "${REMOTE_SNAPSHOTTER_ROOT}snapshotter/snapshots/" \
72+
-maxdepth 1 -mindepth 1 -type d -exec umount "{}/fs" \;
73+
fi
74+
rm -rf "${REMOTE_SNAPSHOTTER_ROOT}"*
75+
}
76+
77+
echo "cleaning up the environment..."
78+
kill_all "containerd"
79+
kill_all "containerd-stargz-grpc"
80+
kill_all "scrape"
81+
if [ "$NOCLEANUP" == "-nocleanup" ]; then
82+
echo "DO NOT CLEANUP containerd & stargz-snapshotter layers"
83+
else
84+
cleanup
85+
fi
86+
if [ "${NOSNAPSHOTTER}" == "-nosnapshotter" ] ; then
87+
echo "DO NOT RUN remote snapshotter"
88+
else
89+
echo "running remote snaphsotter..."
90+
containerd-stargz-grpc --log-level=debug \
91+
--address="${REMOTE_SNAPSHOTTER_SOCKET}" \
92+
--config="${REMOTE_SNAPSHOTTER_CONFIG_DIR}config.stargz.toml" \
93+
&>/var/log/containerd-stargz-grpc.log &
94+
retry ls "${REMOTE_SNAPSHOTTER_SOCKET}"
95+
mkdir -p "$(dirname "${SCRAPE_OUTPUT_FILENAME}")"
96+
scrape -output "${SCRAPE_OUTPUT_FILENAME}" -netinf eth0 -interval 1s \
97+
-pid "$(ps aux | grep -v grep | grep containerd-stargz-grpc | awk '{print $2}')" &
98+
fi
99+
echo "running containerd..."
100+
containerd --log-level debug \
101+
--config="${CONTAINERD_CONFIG_DIR}config.containerd.toml" &>/var/log/containerd.log &
102+
retry ctr version

0 commit comments

Comments
 (0)