diff --git a/kola/tests/bpf/bpf.go b/kola/tests/bpf/bpf.go deleted file mode 100644 index d26b783e4..000000000 --- a/kola/tests/bpf/bpf.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright The Mantle Authors. -// SPDX-License-Identifier: Apache-2.0 -package bpf - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" - "time" - - "github.com/coreos/pkg/capnslog" - "github.com/flatcar/mantle/kola/cluster" - "github.com/flatcar/mantle/kola/register" - "github.com/flatcar/mantle/util" -) - -// cmdPrefix is a temporary hack to pull `bcc` tools into Flatcar -const cmdPrefix = "docker run -d --name %s -v /lib/modules:/lib/modules -v /sys/kernel/debug:/sys/kernel/debug -v /sys/fs/cgroup:/sys/fs/cgroup -v /sys/fs/bpf:/sys/fs/bpf --privileged --net host --pid host ghcr.io/flatcar/bcc %s" - -var ( - plog = capnslog.NewPackageLogger("github.com/flatcar/mantle", "kola/tests/bpf") -) - -// Log defines the standard log format -// from Docker -// https://docs.docker.com/config/containers/logging/json-file/ -type Log struct { - Log string `json:"log"` - Time string `json:"time"` - Stream string `json:"stream"` -} - -func init() { - register.Register(®ister.Test{ - Run: execsnoopTest, - ClusterSize: 1, - Name: `bpf.execsnoop`, - Distros: []string{"cl"}, - // required while SELinux policy is not correcly updated to support - // `bpf` and `perfmon` permission. - Flags: []register.Flag{register.NoEnableSelinux}, - // This test is normally not related to the cloud environment - Platforms: []string{"qemu", "qemu-unpriv", "azure"}, - }) -} - -func execsnoopTest(c cluster.TestCluster) { - m := c.Machines()[0] - containerName := "execsnoop" - - // filter commands with `docker ps` - plog.Infof("running %s container", containerName) - cmd := fmt.Sprintf(cmdPrefix, containerName, "/usr/share/bcc/tools/execsnoop -n docker -l ps") - if _, err := c.SSH(m, cmd); err != nil { - c.Fatalf("unable to run SSH command '%s': %v", cmd, err) - } - - // wait for the container and the `execsnoop` command to be correctly started before - // generating traffic. - if err := util.Retry(10, 2*time.Second, func() error { - - // Run 'docker ps' to trigger log output. Execsnoop won't print anything, not even the header, - // before it's been triggered for the first time. - _ = c.MustSSH(m, "docker ps") - - // we first assert that the container is running and then the process too. - // it's not possible to use `docker top...` command because it's the execsnoop itself who takes some time to start. - logs, err := c.SSH(m, fmt.Sprintf("sudo cat $(docker inspect --format='{{.LogPath}}' %s)", containerName)) - if err != nil { - return fmt.Errorf("getting running process: %w", err) - } - - if len(logs) > 0 { - return nil - } - - return fmt.Errorf("no logs, the service has not started yet properly") - }); err != nil { - c.Fatalf("unable to get container ready: %v", err) - } - - // generate some "traffic" - _ = c.MustSSH(m, "docker info") - _ = c.MustSSH(m, fmt.Sprintf("docker logs %s", containerName)) - _ = c.MustSSH(m, fmt.Sprintf("docker top %s", containerName)) - - plog.Infof("getting logs from %s container", containerName) - if err := util.Retry(10, 2*time.Second, func() error { - logs, err := c.SSH(m, fmt.Sprintf("sudo cat $(docker inspect --format='{{.LogPath}}' %s)", containerName)) - if err != nil { - c.Fatalf("unable to run SSH command: %v", err) - } - dockerLogs := bytes.Split(logs, []byte("\n")) - - // we have the headers of the table - // then 2 lines for docker ps and the torcx call if torcx is used - if len(dockerLogs) < 2 { - return fmt.Errorf("Waiting for execsnoop log entries") - } - - l := Log{} - for _, log := range dockerLogs { - - if err := json.Unmarshal(log, &l); err != nil { - return fmt.Errorf("Transient error unmarshalling docker log: %v", err) - } - if l.Stream == "stderr" { - return fmt.Errorf("Transient error: stream should not log to 'stderr'") - } - if strings.Contains(l.Log, "docker info") || strings.Contains(l.Log, "docker top") || strings.Contains(l.Log, "docker logs") { - c.Fatal("log should not contain 'docker info' or 'docker top' or 'docker logs'") - } - - if strings.Contains(l.Log, "docker ps") { - return nil - } - } - return fmt.Errorf("Waiting for execsnoop log entries") - }); err != nil { - c.Fatalf("Unable to find 'docker ps' log lines in execsnoop logs: %v", err) - } -} diff --git a/kola/tests/bpf/ig.go b/kola/tests/bpf/ig.go new file mode 100644 index 000000000..b856e20cc --- /dev/null +++ b/kola/tests/bpf/ig.go @@ -0,0 +1,100 @@ +// Copyright The Mantle Authors. +// SPDX-License-Identifier: Apache-2.0 +package bpf + +import ( + "fmt" + "strings" + + "github.com/coreos/go-semver/semver" + "github.com/flatcar/mantle/kola" + "github.com/flatcar/mantle/kola/cluster" + "github.com/flatcar/mantle/kola/register" + "github.com/flatcar/mantle/platform/conf" +) + +var igVer = "v0.50.0" + +func init() { + register.Register(®ister.Test{ + Run: igTest, + Name: `bpf.ig`, + Distros: []string{"cl"}, + MinVersion: semver.Version{Major: 4081}, + // This test is normally not related to the cloud environment + Platforms: []string{"qemu", "qemu-unpriv", "azure"}, + // Required while SELinux policy is not correctly updated to support + // `bpf` and `perfmon` permissions. + Flags: []register.Flag{register.NoEnableSelinux}, + }) +} + +func igTest(c cluster.TestCluster) { + arch := strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0] + if arch == "amd64" { + arch = "x86-64" + } + + c.Run("ig", func(c cluster.TestCluster) { + node, err := c.NewMachine(conf.Butane(fmt.Sprintf(` + variant: flatcar + version: 1.0.0 + storage: + files: + - path: /etc/extensions/ig.raw + mode: 0644 + contents: + source: https://extensions.flatcar.org/extensions/ig-%s-%s.raw + `, igVer, arch))) + if err != nil { + c.Fatalf("creating node: %v", err) + } + + // The following test that ig tracing and filtering works when applied + // to the host. ig runs in the foreground, but it can take a few seconds + // to be ready, even after prefetching the gadget. To avoid flakiness, + // ig is put in the background and grep is used to wait for the + // "running" debug message. coproc doesn't handle stderr, so stderr is + // redirected to stdout and the real stdout is redirected to a file for + // later analysis. The timeout against grep ensures that we don't wait + // for "running" forever. The gadget is prefetched with --help so that + // the download does not count against the timeout. The trap prevents ig + // from keeping the script alive when an unexpected error occurs. + + if _, err := c.SSH(node, fmt.Sprintf(` + set -ex + sudo ig run trace_exec:%[1]s --help + coproc IG { sudo ig run trace_exec:%[1]s --host --filter 'proc.comm=docker,args~ps' --output json --verbose 2>&1 > ig.json; } + trap 'kill %%%%' ERR + timeout 30 grep -F -m1 'running...' <&${IG[0]} + docker info + docker ps + docker images + kill %%%% + wait + jq -s -e '.[] | select(.args == "/usr/bin/docker\u00a0ps")' ig.json + jq -s -e 'isempty(.[] | select(.args == "/usr/bin/docker\u00a0info"))' ig.json + jq -s -e 'isempty(.[] | select(.args == "/usr/bin/docker\u00a0images"))' ig.json + `, igVer)); err != nil { + c.Fatalf("ig run trace_exec did not behave as expected: %v", err) + } + + if _, err := c.SSH(node, fmt.Sprintf(` + set -ex + sudo ig run trace_dns:%[1]s --help + coproc IG { sudo ig run trace_dns:%[1]s --host --filter 'name=flatcar.org.' --output json --verbose 2>&1 > ig.json; } + trap 'kill %%%%' ERR + timeout 30 grep -F -m1 'running...' <&${IG[0]} + dig kinvolk.io + dig flatcar.org + dig stable.release.flatcar-linux.net + kill %%%% + wait + jq -s -e '.[] | select(.name == "flatcar.org.")' ig.json + jq -s -e 'isempty(.[] | select(.name == "kinvolk.io."))' ig.json + jq -s -e 'isempty(.[] | select(.name == "stable.release.flatcar-linux.net."))' ig.json + `, igVer)); err != nil { + c.Fatalf("ig run trace_dns did not behave as expected: %v", err) + } + }) +} diff --git a/kola/tests/bpf/local-gadget.go b/kola/tests/bpf/local-gadget.go deleted file mode 100644 index 7b6956f06..000000000 --- a/kola/tests/bpf/local-gadget.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright The Mantle Authors. -// SPDX-License-Identifier: Apache-2.0 -package bpf - -import ( - "bytes" - "strings" - "text/template" - - "github.com/coreos/go-semver/semver" - "github.com/flatcar/mantle/kola" - "github.com/flatcar/mantle/kola/cluster" - "github.com/flatcar/mantle/kola/register" - "github.com/flatcar/mantle/kola/tests/docker" - "github.com/flatcar/mantle/platform/conf" -) - -type gadget struct { - // Arch holds the gadget architecture (arm or amd). - Arch string - // Version holds the version of the gadget (v0.4.1 for example). - Version string - // Sum holds the SHA512 checksum to verify binary. - Sum string - // Cmd holds the command for the given gadget. - Cmd string -} - -var ( - // binaries holds the map of binaries for each supported - // architecture. - binaries = map[string]gadget{ - "arm64": gadget{ - Version: "0.4.1", - Sum: "4b5761dd08afea378e7e58a5a76b76c727ed59d327e042a42dd72330cac7bcd516da7574f69d22a4076c07dcafa639d08aaced56cf624816c5815226b33a0961", - }, - "amd64": gadget{ - Version: "0.4.1", - Sum: "d68c2c55ac3d783f52aaf9b2e360955fa542f48901b6dc7fc4d52cd91e22f5c9839e6dd89e575923f117752e82f052d397c002cdd9acbcc3adfb893b8a18cdd8", - }, - } - config = `storage: - files: - - path: /opt/local-gadget.tar.gz - filesystem: root - mode: 0755 - contents: - remote: - url: https://github.com/kinvolk/inspektor-gadget/releases/download/v{{ .Version }}/local-gadget-linux-{{ .Arch }}.tar.gz - verification: - hash: - function: sha512 - sum: {{ .Sum }} - - path: /opt/local-gadget.cmd - filesystem: root - mode: 0644 - contents: - inline: | - {{ .Cmd }} -systemd: - units: - - name: prepare-local-gadget.service - enabled: true - contents: | - [Unit] - Description=Unpack local-gadget to /opt/bin/ - ConditionPathExists=!/opt/bin/local-gadget - [Service] - Type=oneshot - RemainAfterExit=true - Restart=on-failure - ExecStartPre=/usr/bin/mkdir --parents /opt/bin - ExecStartPre=/usr/bin/tar -v --extract --file /opt/local-gadget.tar.gz --directory /opt/bin --no-same-owner - ExecStart=/usr/bin/rm /opt/local-gadget.tar.gz - [Install] - WantedBy=multi-user.target - - name: local-gadget.service - enabled: true - contents: | - [Unit] - Description=Run local-gadget - After=prepare-local-gadget.service - Requires=prepare-local-gadget.service - [Service] - User=root - Type=fork - RemainAfterExit=true - Restart=on-failure - ExecStart=/opt/bin/local-gadget - StandardInput=file:/opt/local-gadget.cmd - StandardOutput=file:/tmp/local-gadget.res - [Install] - WantedBy=multi-user.target` -) - -func init() { - register.Register(®ister.Test{ - Run: localGadgetTest, - Name: `bpf.local-gadget`, - Distros: []string{"cl"}, - // required while SELinux policy is not correcly updated to support - // `bpf` and `perfmon` permission. - Flags: []register.Flag{register.NoEnableSelinux}, - // current LTS has DOCKER_API_VERSION=1.40 which is too old for local-gadget docker client. - // "client version 1.41 is too new. Maximum supported API version is 1.40" - MinVersion: semver.Version{Major: 3033}, - // This test is normally not related to the cloud environment - Platforms: []string{"qemu", "qemu-unpriv", "azure"}, - SkipFunc: func(version semver.Version, channel, arch, platform string) bool { - // LTS (3033) does not have the network-kargs service pulled in: - // https://github.com/flatcar/coreos-overlay/pull/1848/commits/9e04bc12c3c7eb38da05173dc0ff7beaefa13446 - // Let's skip this test for < 3034 on ESX. - return version.LessThan(semver.Version{Major: 3034}) && platform == "esx" - }, - }) -} - -func localGadgetTest(c cluster.TestCluster) { - arch := strings.SplitN(kola.QEMUOptions.Board, "-", 2)[0] - gadget := binaries[arch] - - gadget.Arch = arch - - tmpl, err := template.New("user-data").Parse(config) - if err != nil { - c.Fatalf("parsing user-data: %v", err) - } - - c.Run("dns gadget", func(c cluster.TestCluster) { - gadget.Cmd = `create dns kola-dns --container-selector=shell01 - list-traces - stream -f kola-dns` - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, gadget); err != nil { - c.Fatalf("rendering user-data: %v", err) - } - - node, err := c.NewMachine(conf.ContainerLinuxConfig(buf.String())) - if err != nil { - c.Fatalf("creating node: %v", err) - } - - docker.GenDockerImage(c, node, "dig", []string{"dig"}) - - if _, err := c.SSH(node, "docker run --rm --name shell01 dig dig flatcar.org"); err != nil { - c.Fatalf("unable to run docker cmd: %v", err) - } - - out, err := c.SSH(node, "grep -m 1 pkt_type /tmp/local-gadget.res | jq -r .name") - if err != nil { - c.Fatalf("unable to get local-gadget res: %v", err) - } - - name := string(out) - - if name != "flatcar.org." { - c.Fatalf("should have 'flatcar.org.', got: %v", err) - } - }) -}