test: Add new sanity check test

Tests if the correct boot partition and version is used.

Author: mattiaswal

board/common/post-build.sh

@@ -86,6 +86,7 @@ rm -f "$TARGET_DIR/etc/os-release"
     fi
 } > "$TARGET_DIR/etc/os-release"
 
+cp "$TARGET_DIR/etc/os-release" "$BINARIES_DIR/"
 echo "$INFIX_TAGLINE $INFIX_VERSION -- $(date +"%b %e %H:%M %Z %Y")" > "$TARGET_DIR/etc/version"
 
 # In case of ambguities, this is what the image was built from

test/case/all.yaml

@@ -6,12 +6,21 @@
   infamy:
     specification: False
 
+- case: meta/sanity-check.py
+  infamy:
+    specification: False
+
 - name: Misc tests
   suite: misc/misc.yaml
 
 - name:  ietf-system
   suite: ietf_system/ietf_system.yaml
 
+# Upgrade may leave wrong boot order
+- case: meta/sanity-check.py
+  infamy:
+    specification: False
+
 - name:  ietf-syslog
   suite: ietf_syslog/ietf_syslog.yaml
 

test/case/meta/reproducible.py

@@ -13,27 +13,4 @@
         else:
             print(f"Specify PYTHONHASHSEED={seed} to reproduce this test environment")
 
-    with test.step("Discover topology and attach to available DUTs"):
-        env = infamy.Env(False)
-        ctrl = env.ptop.get_ctrl()
-
-        duts = {}
-        for ix in env.ptop.get_infixen():
-            cport, ixport = env.ptop.get_mgmt_link(ctrl, ix)
-            print(f"Attaching to {ix}:{ixport} via {ctrl}:{cport}")
-            duts[ix] = env.attach(ix, ixport)
-
-    with test.step("Log running software versions"):
-        for name, tgt in duts.items():
-            sys = tgt.get_data("/ietf-system:system-state")
-            sw = sys["system-state"]["software"]
-            plt = sys["system-state"]["platform"]
-
-            print(f"{name}:")
-            for k,v in plt.items():
-                print(f"  {k:<16s}  {v}")
-
-            for k in ("compatible", "booted"):
-                print(f"  {k:<16s}  {sw[k]}")
-
     test.succeed()

test/case/meta/sanity-check.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+import infamy
+import os
+
+with infamy.Test() as test:
+    with test.step("Discover topology and attach to available DUTs"):
+        env = infamy.Env(False)
+        ctrl = env.ptop.get_ctrl()
+        duts = {}
+        duts_state = {}
+        for ix in env.ptop.get_infixen():
+            cport, ixport = env.ptop.get_mgmt_link(ctrl, ix)
+            print(f"Attaching to {ix}:{ixport} via {ctrl}:{cport}")
+            duts[ix] = env.attach(ix, ixport)
+            sys = duts[ix].get_data("/ietf-system:system-state")
+            sw = sys["system-state"]["software"]
+            plt = sys["system-state"]["platform"]
+            duts_state[ix] = {
+                "platform" : plt,
+                "software" : sw
+            }
+    with test.step("Verify bootorder"):
+        for name, tgt in duts.items():
+            expected = env.ptop.get_boot(name)
+            running  = duts_state[name]["software"]["booted"]
+            print(f"{name}: booted: {running} expected: {expected}")
+
+            if running != expected:
+                test.fail()
+    with test.step("Verify software version"):
+        expected=os.environ.get("VERSION")
+        for name, tgt in duts.items():
+            running=duts_state[name]["platform"]["os-version"]
+            print(f"{name}: booted: {running} expected {expected}")
+            if running != expected:
+                test.fail()
+    test.succeed()

test/env

@@ -67,6 +67,9 @@ usage: test/env [-cDhiKr] -f <IMAGE> -q <QENETH-DIR> <COMMAND> [<ARGS>...]
       topology. If the command is containerized, it will be launched
       in the host's network namespace
 
+    -v <VERSION>
+      Version of the image under test.
+
 EOF
 }
 
@@ -154,7 +157,7 @@ containerize=yes
 files=
 basedir="$ixdir"
 
-while getopts "b:cCDf:hiKp:q:rt:" opt; do
+while getopts "b:cCDf:hiKp:q:rt:v:" opt; do
     case ${opt} in
 	b)
 	    basedir="$OPTARG"
@@ -198,6 +201,9 @@ while getopts "b:cCDf:hiKp:q:rt:" opt; do
 	    network="--network host --volume $topology:$topology:ro"
 	    runners="docker podman"
 	    ;;
+	v)
+	    version="$OPTARG"
+	    ;;
 	*)
 	    >&2 echo "Unknown option -$opt"
 	    usage
@@ -234,6 +240,7 @@ if [ "$containerize" ]; then
 	 --env NINEPM_PROJ_CONFIG="$NINEPM_PROJ_CONFIG" \
 	 --env QENETH_PATH="$testdir/templates:$testdir" \
 	 --env PS1="$(build_ps1)" \
+	 --env VERSION="$version" \
 	 $extra_env \
 	 --expose 9001-9010 --publish-all \
 	 -v "$HOME/.infix/.ash_history":/root/.ash_history \

test/infamy/topology.py

@@ -131,6 +131,13 @@ def get_password(self, node):
 
         return qstrip(password) if password is not None else "admin"
 
+    def get_boot(self, node):
+        n = self.dotg.get_node(node)
+        b = n[0] if n else {}
+        boot = b.get("boot")
+
+        return _qstrip(boot)
+
     def get_link(self, src, dst, flt=lambda _: True):
         es = self.g.get_edge_data(src, dst)
         for e in es.values():
@@ -150,7 +157,6 @@ def get_ctrl(self):
     def get_infixen(self):
         return self.get_nodes(lambda _, attrs: compatible(attrs, {"requires": {"infix"}}))
 
-
     def get_attr(self, name, default=None):
         return _qstrip(self.dotg.get_attributes().get(name, default))
 

test/test.mk

@@ -14,8 +14,8 @@ mode-qeneth := -q $(test-dir)/virt/quad
 mode-host   := -t $(or $(TOPOLOGY),/etc/infamy.dot)
 mode-run    := -t $(BINARIES_DIR)/qemu.dot
 mode        := $(mode-$(TEST_MODE))
-
-INFIX_IMAGE_ID := $(call qstrip,$(INFIX_IMAGE_ID))
+INFIX_VERSION    := $(shell sed -n 's/^VERSION="\(.*\)"/\1/p' $(BINARIES_DIR)/os-release)
+INFIX_IMAGE_ID   := $(call qstrip,$(INFIX_IMAGE_ID))
 binaries-$(ARCH) := $(addprefix $(INFIX_IMAGE_ID),.img -disk.qcow2)
 pkg-$(ARCH)      := -p $(O)/images/$(addprefix $(INFIX_IMAGE_ID),.pkg)
 binaries-x86_64  += OVMF.fd
@@ -30,7 +30,7 @@ test:
 	$(test-dir)/env -r $(base) $(mode) $(binaries) $(pkg-$(ARCH)) $(ninepm) -v $(TESTS)
 
 test-sh:
-	$(test-dir)/env $(base) $(mode) $(binaries) $(pkg-$(ARCH)) -i /bin/sh
+	$(test-dir)/env $(base) $(mode) $(binaries) $(pkg-$(ARCH)) -v $(INFIX_VERSION) -i /bin/sh
 
 test-spec:
 	@esc_infix_name="$(echo $(INFIX_NAME) | sed 's/\//\\\//g')"; \

test/virt/dual/topology.dot.in

@@ -25,6 +25,7 @@ graph "dual" {
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 } | dut1 | { <e4> e4 | <e5> e5 | <e6> e6 }",
 	    pos="10,18!",
 	    provides="infix",
+	    boot="primary",
 	    qn_console=9001,
 	    qn_mem="384M",
 	    qn_usb="dut1.usb"
@@ -33,6 +34,7 @@ graph "dual" {
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 } | dut2 | { <e4> e4 | <e5> e5 | <e6> e6 }",
 	    pos="10,12!",
 	    provides="infix",
+	    boot="primary",
 	    qn_console=9002,
 	    qn_mem="384M",
 	    qn_usb="dut2.usb"

test/virt/quad/topology.dot.in

@@ -24,6 +24,7 @@ graph "quad" {
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut1 | { <e5> e5 | <e6> e6 | <e7> e7 | <e8> e8}",
 	    pos="10,30!",
 	    provides="infix",
+	    boot="primary",
 	    qn_console=9001,
 	    qn_mem="384M",
 	    qn_usb="dut1.usb"
@@ -32,6 +33,7 @@ graph "quad" {
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut2 | { <e5> e5 | <e6> e6  | <e7> e7 | <e8> e8}",
 	    pos="0,20!",
 	    provides="infix",
+	    boot="primary",
 	    qn_console=9002,
 	    qn_mem="384M",
 	    qn_usb="dut2.usb"
@@ -40,6 +42,7 @@ graph "quad" {
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut3 | { <e5> e5 | <e6> e6  | <e7> e7 | <e8> e8}",
 	    pos="0,10!",
 	    provides="infix",
+	    boot="primary",
 	    qn_console=9003,
 	    qn_mem="384M",
 	    qn_usb="dut3.usb"
@@ -49,6 +52,7 @@ graph "quad" {
 	    label="{ <e1> e1 | <e2> e2 | <e3> e3 | <e4> e4 } | dut4 | { <e5> e5 | <e6> e6  | <e7> e7 | <e8> e8}",
 	    pos="10,0!",
 	    provides="infix",
+	    boot="primary",
 	    qn_console=9004,
 	    qn_mem="384M",
 	    qn_usb="dut4.usb"