Merge pull request #1088 from nplanel/run-func-tests-in-vagrant

CI : run functional tests in vagrant

Author: lebauce

contrib/ansible/roles/skydive_agent/tasks/docker.yml

@@ -10,6 +10,34 @@
     state: started
     enabled: yes
 
+- name: Run docker registry
+  shell: "docker run -d -p 5000:5000 --restart=always --name registry registry:2"
+  when: '"file://" in skydive_binary_remote_location'
+
+- name: Set facts
+  set_fact: "skydive_docker_registry=localhost:5000"
+  when: '"file://" in skydive_binary_remote_location'
+
+- name: Copy docker image
+  copy:
+    src: "{{ skydive_binary_remote_location | replace('file://', '') }}"
+    dest: /tmp
+    force: true
+    mode: 0755
+  when: '"file://" in skydive_binary_remote_location'
+
+- name: Import skydive image
+  shell: "docker load -i /tmp/{{ skydive_binary_remote_location | replace('file://', '') | basename }}"
+  when: '"file://" in skydive_binary_remote_location'
+
+- name: Tag skydive image
+  shell: "docker tag skydive/skydive:devel {{ skydive_docker_registry }}/{{ skydive_agent_docker_image }}:{{ skydive_docker_image_tag }}"
+  when: '"file://" in skydive_binary_remote_location'
+
+- name: Push skydive image to registry
+  shell: "docker push {{ skydive_docker_registry }}/{{ skydive_agent_docker_image }}:{{ skydive_docker_image_tag }}"
+  when: '"file://" in skydive_binary_remote_location'
+
 - name: Create systemd unit file
   include_role:
     name: skydive_common
@@ -22,7 +50,7 @@
         --privileged --net=host --pid=host \
         -v /var/run/openvswitch/db.sock:/var/run/openvswitch/db.sock \
         -v /var/run/docker.sock:/var/run/docker.sock \
-        -v /var/run/netns:/host/run \
+        -v /var/run/netns:/host/run:ro,shared \
         -v /etc/skydive/skydive.yml:/etc/skydive.yml \
         -e SKYDIVE_NETNS_RUN_PATH=/host/run {{ skydive_agent_docker_extra_env }} \
         -p {{ skydive_agent_port }}:{{ skydive_agent_port }} \

contrib/ansible/roles/skydive_common/tasks/config.yml

@@ -12,7 +12,7 @@
     force: no
   ignore_errors: yes
 
-- name: Create empty config file is needed
+- name: Create empty config file if needed
   copy:
     content: ""
     dest: "{{ skydive_config_file }}"

contrib/vagrant/Vagrantfile

@@ -22,6 +22,16 @@ systemctl start openvswitch
 systemctl start docker
 systemctl enable openvswitch
 systemctl enable docker
+yum -y install libpcap libselinux-python bridge-utils net-tools yum-plugin-copr.noarch
+#curl -o /etc/yum.repos.d/_copr_ganto-lxc3.repo https://copr.fedorainfracloud.org/coprs/ganto/lxc3/repo/epel-7/ganto-lxc3-epel-7.repo
+#yum repolist
+#yum -y install lxd
+#grep -q 'root:1000000:65536' /etc/subuid || echo 'root:1000000:65536' >> /etc/subuid
+#grep -q 'root:1000000:65536' /etc/subgid || echo 'root:1000000:65536' >> /etc/subgid
+#systemctl start lxd
+#ln -s /var/run/lxd.socket /var/lib/lxd/unix.socket
+#systemctl enable lxd
+#lxd init --auto
 SCRIPT
 
 $skydive_extra_config = {
@@ -30,6 +40,18 @@ $skydive_extra_config = {
   "agent.topology.netlink.metrics_update" => 5,
 }
 
+if DEVMODE then
+  # default config from tests/tests.go:testConfig
+  $skydive_extra_config["flow.expire"] = "600"
+  $skydive_extra_config["flow.update"] = "10"
+  $skydive_extra_config["agent.topology.probes"] = "netlink netns ovsdb docker"
+  $skydive_extra_config["agent.topology.metrics_update"] = "5"
+  $skydive_extra_config["agent.metadata.mydict.value"] = "123"
+  $skydive_extra_config["analyzer.startup.capture_gremlin"] = "g.V().Has('Name','startup-vm2')"
+  $skydive_extra_config["ovs.oflow.enable"] = "true"
+  $skydive_extra_config["logging.level"] = "DEBUG"
+end
+
 def ifcfg_config(interface, network, netmask, ip)
   ifcfg_template = <<-IFCFG
 cat > /etc/sysconfig/network-scripts/ifcfg-#{interface} <<EOF
@@ -69,9 +91,8 @@ def provision_es(vm, hosts)
     ansible.playbook = "playbook-es.yml"
     ansible.groups = { "elasticsearch": hosts.map { |h| h["name"] } }
     ansible.limit = "all"
-    ansible.verbose = "-vvv"
     ansible.galaxy_role_file = 'requirements.yml'
-    ansible.sudo = true
+    ansible.become = true
   end
 
   $skydive_extra_config["analyzer.flow.backend"] = "elasticsearch"

contrib/vagrant/setup-common.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
-sudo yum -y install python2 python-yaml
+sudo yum -y install python2 python-yaml libpcap ntpdate audit
 sudo setenforce 0
-sudo sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/sysconfig/selinux
+sudo sed -i 's/SELINUX=enforcing/SELINUX=permissive/' /etc/sysconfig/selinux

gremlin/query.go

@@ -24,6 +24,7 @@ package gremlin
 
 import (
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/skydive-project/skydive/common"
@@ -104,6 +105,16 @@ func (q QueryString) Context(list ...interface{}) QueryString {
 	for _, v := range list {
 		if !first {
 			newQ = newQ.appends(", ")
+		} else {
+			switch t := v.(type) {
+			case string:
+				up := strings.ToUpper(t)
+				if up == "NOW" || up == "FOREVER" {
+					newQ = newQ.appends(up)
+					first = false
+					continue
+				}
+			}
 		}
 		first = false
 		switch t := v.(type) {

scripts/ci/run-vagrant-tests.sh

@@ -12,51 +12,116 @@ set -v
 
 dir="$(dirname "$0")"
 
-cd ${GOPATH}/src/github.com/skydive-project/skydive
+root=${GOPATH}/src/github.com/skydive-project/skydive
+cd $root
+
+if [ "$DEVMODE" = "true" ]; then
+    make
 
-if [ "$DEVMODE" == "true" ]; then
-    make static
     make srpm
     VERSION=$(make -s version | cut -d '-' -f 1)
     TAG=$(make -s version | cut -s -d '-' -f 2- | tr '-' '.')
     mock -r centos-7-x86_64 -D "fullver $(make -s version)" --resultdir . --rebuild rpmbuild/SRPMS/skydive-${VERSION}*${TAG}.src.rpm
     mkdir -p rpmbuild/RPMS/{x86_64,noarch}
     mv skydive*-${VERSION}-*${TAG}*.x86_64.rpm rpmbuild/RPMS/x86_64/
     mv skydive*-${VERSION}-*${TAG}*.noarch.rpm rpmbuild/RPMS/noarch/
+
     make docker-image
     docker save skydive/skydive:devel -o skydive-docker-devel.tar
+
+    make test.functionals.compile
 fi
 
 cd contrib/vagrant
 
+export ANALYZER_COUNT=1
+export AGENT_COUNT=1
+export SKYDIVE_RELEASE=master
+
 function vagrant_cleanup {
+    set +e
+    echo "===== journalctl agent1"
+    vagrant ssh agent1 -c 'sudo journalctl -xe | grep skydive'
+    echo "===== journalctl analyzer1"
+    vagrant ssh analyzer1 -c 'sudo journalctl -xe | grep skydive'
     vagrant destroy --force
 }
 trap vagrant_cleanup EXIT
 
-export ANALYZER_COUNT=2
-export AGENT_COUNT=1
-export SKYDIVE_RELEASE=master
+function run_functional_tests {
+  vagrant ssh-config > vagrant-ssh-config
+  scp -F vagrant-ssh-config $root/tests/functionals agent1:
+  rsync -av -e 'ssh -F vagrant-ssh-config' $root/tests/pcaptraces agent1:
+  AGENT1_IP=$(vagrant ssh-config agent1 | grep HostName | awk '{print $2}')
+  ANALYZER1_IP=$(vagrant ssh-config analyzer1 | grep HostName | awk '{print $2}')
+
+  vagrant ssh agent1 -c 'for i in $(find /proc/sys/net/bridge/ -type f) ; do echo 0 | sudo tee $i ; done'
+  vagrant ssh agent1 -c 'sudo iptables -F ; sudo iptables -P FORWARD ACCEPT'
+
+  if [ "$mode" = "container" ]; then
+      OPT="-nooftests"
+  fi
+  vagrant ssh agent1 -c "AGENT1_IP=$AGENT1_IP SKYDIVE_ANALYZERS=\"$ANALYZER1_IP:8082\" sudo -E ./functionals -agenttestsonly -test.v $OPT"
+
+  if [ "$mode" = "package" ]; then
+      for a in analyzer1 agent1; do
+          echo "===== ausearch AVC on $a ======"
+          vagrant ssh $a -c 'sudo ausearch -m avc -r' || true
+      done
+  fi
+  rm -f vagrant-ssh-config
+}
+
+function install_skydive_selinux_enforcing {
+    cat <<'EOF' | vagrant ssh $1 -- bash -
+sudo setenforce 1
+sudo sed -i 's/SELINUX=permissive/SELINUX=enforcing/' /etc/sysconfig/selinux
+EOF
+}
+
+function install_skydive_from_docker_image {
+    cat <<'EOF' | vagrant ssh $1 -- bash -
+t=$(mktemp -d skydive.docker.XXX)
+pushd $t
+sudo docker image save skydive/skydive | tar xf - "*/layer.tar"
+find . -name "layer.tar" | xargs -n 1 --replace=XXX sudo tar -C / -xf XXX usr/bin/skydive &>/dev/null || true
+popd
+rm -rf $t
+EOF
+}
 
 for mode in $MODES
 do
+  echo "================== deploying mode $mode ==============================="
   DEPLOYMENT_MODE=$mode vagrant box update
   DEPLOYMENT_MODE=$mode vagrant up --provision-with common
   DEPLOYMENT_MODE=$mode vagrant provision
 
+  vagrant ssh analyzer1 -- sudo ntpdate fr.pool.ntp.org
+  vagrant ssh agent1 -- sudo ntpdate fr.pool.ntp.org
+
   vagrant ssh analyzer1 -- sudo cat /etc/skydive/skydive.yml
-  vagrant ssh analyzer2 -- sudo cat /etc/skydive/skydive.yml
 
   vagrant ssh analyzer1 -- sudo journalctl -n 100 -u skydive-analyzer
-  vagrant ssh analyzer2 -- sudo journalctl -n 100 -u skydive-analyzer
   vagrant ssh agent1 -- sudo journalctl -n 100 -u skydive-agent
 
   vagrant ssh analyzer1 -- curl http://localhost:8082
+
+  if [ "$mode" = "container" ]; then
+      install_skydive_from_docker_image analyzer1
+      install_skydive_from_docker_image agent1
+  fi
+
+  if [ "$mode" = "package" ]; then
+      install_skydive_selinux_enforcing analyzer1
+      install_skydive_selinux_enforcing agent1
+  fi
+
+  vagrant ssh analyzer1 -c 'set -e; skydive client query "g.V()"'
+
   if [ "$mode" != "container" ]; then
-      vagrant ssh analyzer1 -c 'set -e; skydive client query "g.V()"'
-  else
-      CONTAINER=$(vagrant ssh analyzer1 -- sudo docker ps | grep 'skydive/skydive:latest' | awk '{print $1}')
-      vagrant ssh analyzer1 -- sudo docker exec -t $CONTAINER skydive client query "'g.V()'"
+      sleep 10
+      run_functional_tests
   fi
 
   vagrant destroy --force

tests/alert_test.go

@@ -86,6 +86,11 @@ func TestAlertWebhook(t *testing.T) {
 
 	testPassed.Store(false)
 
+	agent1IP := os.Getenv("AGENT1_IP")
+	if agent1IP == "" {
+		agent1IP = "localhost"
+	}
+
 	ListenAndServe := func(addr string, port int) {
 		listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, port))
 		if err != nil {
@@ -123,14 +128,14 @@ func TestAlertWebhook(t *testing.T) {
 
 		setupFunction: func(c *TestContext) error {
 			wg.Add(1)
-			ListenAndServe("localhost", 8080)
+			ListenAndServe(agent1IP, 8080)
 
 			alertLock.Lock()
 			defer alertLock.Unlock()
 
 			al = types.NewAlert()
 			al.Expression = "G.V().Has('Name', 'alert-ns-webhook', 'Type', 'netns')"
-			al.Action = "http://localhost:8080/"
+			al.Action = fmt.Sprintf("http://%s:8080/", agent1IP)
 
 			if err = c.client.Create("alert", al); err != nil {
 				return fmt.Errorf("Failed to create alert: %s", err.Error())
@@ -165,6 +170,10 @@ func TestAlertWebhook(t *testing.T) {
 }
 
 func TestAlertScript(t *testing.T) {
+	if helper.AgentTestsOnly {
+		t.Skip("this test works only when agent and analyzers are on the same host")
+	}
+
 	var (
 		err        error
 		al         *types.Alert
@@ -260,7 +269,7 @@ func TestAlertWithTimer(t *testing.T) {
 		},
 
 		setupFunction: func(c *TestContext) error {
-			ws, err = helper.WSConnect(config.GetString("analyzer.listen"), 5, nil)
+			ws, err = helper.WSConnect(config.GetStringSlice("analyzers")[0], 5, nil)
 			if err != nil {
 				return err
 			}
@@ -334,7 +343,7 @@ func TestMultipleTriggering(t *testing.T) {
 		},
 
 		setupFunction: func(c *TestContext) error {
-			ws, err = helper.WSConnect(config.GetString("analyzer.listen"), 5, nil)
+			ws, err = helper.WSConnect(config.GetStringSlice("analyzers")[0], 5, nil)
 			if err != nil {
 				return err
 			}