Skip to content

Commit df1a4b7

Browse files
ginglis13ginglis13
authored
Merge pull request #576 from ginglis13/critest-buildkite
Add critest as step to buildkite pipeline.
2 parents d1c2a0a + 2214a0f commit df1a4b7

File tree

9 files changed

+127
-45
lines changed

9 files changed

+127
-45
lines changed

‎.buildkite/pipeline.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,15 @@ steps:
108108
- make -C examples integ-test TEST_POOL=build_${BUILDKITE_BUILD_NUMBER}_example
109109
timeout_in_minutes: 10
110110

111+
- label: ":rotating_light: cri conformance tests"
112+
agents:
113+
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
114+
distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
115+
hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
116+
env:
117+
DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
118+
FICD_DM_VOLUME_GROUP: fcci-vg
119+
command:
120+
- make -C runtime critest FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_critest
121+
timeout_in_minutes: 10
122+

‎runtime/Makefile

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,14 +150,20 @@ critest:
150150
--volume /dev:/dev \
151151
--volume /run/udev/control:/run/udev/control \
152152
--volume $(CURDIR)/..:/src \
153+
--volume $(CURDIR)/../examples/etc/containerd/firecracker-runtime.json:/etc/containerd/firecracker-runtime.json \
154+
--volume $(CURDIR)/../tools/demo/fc-br0.interface:/etc/network/interfaces.d/fc-br0 \
155+
--volume $(CURDIR)/logs:/var/log/firecracker-containerd-test \
156+
--volume $(CURDIR)/../tools/critest:/src/runtime/critest \
153157
--volume $(GO_CACHE_VOLUME_NAME):/go \
154158
--env FICD_DM_VOLUME_GROUP=$(FICD_DM_VOLUME_GROUP) \
155159
--env FICD_DM_POOL=$(FICD_DM_POOL) \
156160
--env GOPROXY=direct \
157161
--env GOSUMDB=off \
162+
--env ACK_GINKGO_DEPRECATIONS=1.16.5 \
158163
--workdir="/src/runtime" \
159164
$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \
160-
"critest -runtime-endpoint unix:///run/firecracker-containerd/containerd.sock"
165+
"sleep 1 && critest -ginkgo.noColor -runtime-endpoint unix:///run/firecracker-containerd/containerd.sock | \
166+
./critest/critest_diff.sh"
161167

162168
clean:
163169
- rm -f containerd-shim-aws-firecracker

‎tools/critest/critest_diff.sh

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#! /bin/bash
2+
#
3+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License"). You may
6+
# not use this file except in compliance with the License. A copy of the
7+
# License is located at
8+
#
9+
# http://aws.amazon.com/apache2.0/
10+
#
11+
# or in the "license" file accompanying this file. This file is distributed
12+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
13+
# express or implied. See the License for the specific language governing
14+
# permissions and limitations under the License.
15+
16+
17+
# Create temporary critest output file
18+
critest_output="$(</dev/stdin)"
19+
critest_output_file="$(mktemp)"
20+
echo "$critest_output" >> "$critest_output_file"
21+
22+
set -eu
23+
24+
# Remove up until report summary
25+
sed -i -E '0,/^Summarizing [0-9][0-9]? Failure[s]?:$/d' "$critest_output_file" # Remove empty lines
26+
sed -i '/^$/d' "$critest_output_file"
27+
28+
# Remove unnecessary error messages
29+
sed -i '/^\/.*[0-9]$/d' "$critest_output_file"
30+
sed -i '/^Ran [0-9][0-9] of [0-9][0-9] Specs in .*seconds$/d' "$critest_output_file"
31+
sed -i '/^--- FAIL: TestCRISuite.*$/d' "$critest_output_file"
32+
sed -i '/^FAIL.*$/d' "$critest_output_file"
33+
sed -i '/^Ran.*$/d' "$critest_output_file"
34+
35+
# Diff expected vs. actual
36+
diff -y <(sort critest/expected_critest_output.out) <(sort "$critest_output_file")

‎tools/critest/expected_critest_output.out

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] runtime should support an seccomp profile that blocks setting hostname with SYS_ADMIN
2+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] should support seccomp localhost/profile on the container
3+
[Fail] [k8s.io] Container runtime should support adding volume and device [BeforeEach] runtime should support starting container with volume [Conformance]
4+
[Fail] [k8s.io] Security Context bucket [It] runtime should support that ReadOnlyRootfs is false
5+
[Fail] [k8s.io] PodSandbox runtime should support basic operations on PodSandbox [It] runtime should support running PodSandbox [Conformance]
6+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support HostIpc is false
7+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support ContainerPID
8+
[Fail] [k8s.io] Security Context bucket [It] runtime should return error if RunAsGroup is set without RunAsUser
9+
[Fail] [k8s.io] Security Context NoNewPrivs [BeforeEach] should allow privilege escalation when false
10+
[Fail] [k8s.io] Networking runtime should support networking [It] runtime should support set hostname [Conformance]
11+
[Fail] [k8s.io] Security Context bucket [It] runtime should support dropping ALL capabilities
12+
[Fail] [k8s.io] Security Context bucket [It] runtime should support dropping capability
13+
[Fail] [k8s.io] Security Context SeccompProfilePath docker/default [It] runtime should support setting hostname with docker/default seccomp profile and SYS_ADMIN
14+
[Fail] [k8s.io] Security Context bucket [It] runtime should support ReadonlyPaths
15+
[Fail] [k8s.io] Security Context bucket [It] runtime should support MaskedPaths
16+
[Fail] [k8s.io] PodSandbox runtime should support basic operations on PodSandbox [It] runtime should support removing PodSandbox [Conformance]
17+
[Fail] [k8s.io] Container Mount Propagation runtime should support mount propagation [BeforeEach] mount with 'rshared' should support propagation from host to container and vice versa
18+
[Fail] [k8s.io] Security Context bucket [It] runtime should support that ReadOnlyRootfs is true
19+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support HostPID
20+
[Fail] [k8s.io] Security Context bucket [It] runtime should support RunAsUserName
21+
[Fail] [k8s.io] Container runtime should support log [BeforeEach] runtime should support starting container with log [Conformance]
22+
[Fail] [k8s.io] Container Mount Propagation runtime should support mount propagation [BeforeEach] mount with 'rslave' should support propagation from host to container
23+
[Fail] [k8s.io] Security Context bucket [It] runtime should support SupplementalGroups
24+
[Fail] [k8s.io] Security Context bucket [It] runtime should support Privileged is true
25+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support HostNetwork is true
26+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] runtime should not support a custom seccomp profile without using localhost/ as a prefix
27+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] runtime should ignore a seccomp profile that blocks setting hostname when privileged
28+
[Fail] [k8s.io] Streaming runtime should support streaming interfaces [It] runtime should support exec with tty=false and stdin=false [Conformance]
29+
[Fail] [k8s.io] Multiple Containers [Conformance] when running multiple containers in a pod [BeforeEach] should support network
30+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support stopping container [Conformance]
31+
[Fail] [k8s.io] Multiple Containers [Conformance] when running multiple containers in a pod [BeforeEach] should support container exec
32+
[Fail] [k8s.io] Security Context NoNewPrivs [BeforeEach] should not allow privilege escalation when true
33+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support execSync [Conformance]
34+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support execSync with timeout [Conformance]
35+
[Fail] [k8s.io] Streaming runtime should support streaming interfaces [It] runtime should support attach [Conformance]
36+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support starting container [Conformance]
37+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] runtime should not block setting host name with unconfined seccomp and SYS_ADMIN
38+
[Fail] [k8s.io] PodSandbox runtime should support sysctls [It] should support unsafe sysctls
39+
[Fail] [k8s.io] PodSandbox runtime should support basic operations on PodSandbox [It] runtime should support stopping PodSandbox [Conformance]
40+
[Fail] [k8s.io] Streaming runtime should support streaming interfaces [It] runtime should support exec with tty=true and stdin=true [Conformance]
41+
[Fail] [k8s.io] Security Context SeccompProfilePath docker/default [It] runtime should block sethostname with docker/default seccomp profile and no extra caps
42+
[Fail] [k8s.io] Security Context SeccompProfilePath docker/default [It] should support seccomp docker/default on the container
43+
[Fail] [k8s.io] Networking runtime should support networking [It] runtime should support DNS config [Conformance]
44+
[Fail] [k8s.io] Security Context bucket [It] runtime should support RunAsUser
45+
[Fail] [k8s.io] Container runtime should support log [BeforeEach] runtime should support reopening container log [Conformance]
46+
[Fail] [k8s.io] Container runtime should support adding volume and device [BeforeEach] runtime should support starting container with volume when host path is a symlink [Conformance]
47+
[Fail] [k8s.io] Security Context bucket [It] runtime should support adding capability
48+
[Fail] [k8s.io] Multiple Containers [Conformance] when running multiple containers in a pod [BeforeEach] should support container log
49+
[Fail] [k8s.io] Streaming runtime should support streaming interfaces [It] runtime should support portforward [Conformance]
50+
[Fail] [k8s.io] Container Mount Propagation runtime should support mount propagation [BeforeEach] mount with 'rprivate' should not support propagation
51+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support creating container [Conformance]
52+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support removing running container [Conformance]
53+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support PodPID
54+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support listing container stats [Conformance]
55+
[Fail] [k8s.io] Networking runtime should support networking [It] runtime should support port mapping with host port and container port [Conformance]
56+
[Fail] [k8s.io] Security Context bucket [It] runtime should support RunAsGroup
57+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] should support seccomp unconfined on the container
58+
[Fail] [k8s.io] PodSandbox runtime should support sysctls [It] should support safe sysctls
59+
[Fail] [k8s.io] Security Context bucket [It] runtime should support adding ALL capabilities
60+
[Fail] [k8s.io] Security Context SeccompProfilePath [It] should support seccomp default which is unconfined on the container
61+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support removing stopped container [Conformance]
62+
[Fail] [k8s.io] Security Context bucket [It] runtime should support Privileged is false
63+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support HostIpc is true
64+
[Fail] [k8s.io] Security Context NamespaceOption [It] runtime should support HostNetwork is false
65+
[Fail] [k8s.io] Streaming runtime should support streaming interfaces [It] runtime should support portforward in host network
66+
[Fail] [k8s.io] Networking runtime should support networking [It] runtime should support port mapping with only container port [Conformance]
67+
[Fail] [k8s.io] Container runtime should support basic operations on container [BeforeEach] runtime should support removing created container [Conformance]

‎tools/demo/fcnet.conflist

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"cniVersion": "0.4.0",
2+
"cniVersion": "1.0.0",
33
"name": "fcnet",
44
"plugins": [
55
{
@@ -21,6 +21,9 @@
2121
},
2222
{
2323
"type": "tc-redirect-tap"
24+
},
25+
{
26+
"type": "loopback"
2427
}
2528
]
2629
}

‎tools/docker/Dockerfile.integ-test

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,7 @@ RUN mkdir -p \
3434
# installs so we can minimize re-runs of the time-expensive downloading of images.
3535
COPY tools/docker/config.toml /etc/containerd/config.toml
3636
COPY tools/docker/do_not_edit_for_firecracker-control.config.json /etc/containerd/firecracker-runtime.json
37-
COPY tools/docker/critest/10-mynet.conf /etc/cni/net.d
38-
COPY tools/docker/critest/99-loopback.conf /etc/cni/net.d
37+
COPY tools/demo/fcnet.conflist /etc/cni/net.d/10-fcnet.conflist
3938

4039
RUN --mount=type=bind,source=firecracker-control/cmd/containerd,target=/src \
4140
make -C /src install && \

‎tools/docker/critest/10-mynet.conf

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎tools/docker/critest/99-loopback.conf

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎tools/docker/entrypoint.sh

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,38 +23,17 @@ EOF
2323
cat > /etc/containerd/cri/criconfig.toml <<EOF
2424
version = 2
2525
[plugins]
26-
# The 'plugins."io.containerd.grpc.v1.cri"' table contains all of the server options.
2726
[plugins."io.containerd.grpc.v1.cri"]
28-
29-
# 'plugins."io.containerd.grpc.v1.cri".containerd' contains config related to containerd
3027
[plugins."io.containerd.grpc.v1.cri".containerd]
31-
32-
# snapshotter is the snapshotter used by containerd.
3328
snapshotter = "devmapper"
34-
35-
# default_runtime_name is the default runtime name to use.
3629
default_runtime_name = "containerd-shim-aws-firecracker"
3730
38-
# 'plugins."io.containerd.grpc.v1.cri".containerd.runtimes' is a map from CRI RuntimeHandler strings, which specify types
39-
# of runtime configurations, to the matching configurations.
40-
# In this example, 'runc' is the RuntimeHandler string to match.
4131
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.containerd-shim-aws-firecracker]
42-
# runtime_type is the runtime type to use in containerd.
43-
# The default value is "io.containerd.runc.v2" since containerd 1.4.
44-
# The default value was "io.containerd.runc.v1" in containerd 1.3, "io.containerd.runtime.v1.linux" in prior releases.
4532
runtime_type = "aws.firecracker"
46-
47-
48-
# conf_dir is the directory in which the admin places a CNI conf.
49-
# this allows a different CNI conf for the network stack when a different runtime is being used.
5033
cni_conf_dir = "/etc/cni/net.d"
5134
52-
# 'plugins."io.containerd.grpc.v1.cri".cni' contains config related to cni
5335
[plugins."io.containerd.grpc.v1.cri".cni]
54-
# bin_dir is the directory in which the binaries for the plugin is kept.
5536
bin_dir = "/opt/cni/bin"
56-
57-
# conf_dir is the directory in which the admin places a CNI conf.
5837
conf_dir = "/etc/cni/net.d"
5938
6039
[debug]

0 commit comments

Comments
 (0)