kola/multipath: wait for systemd firstboot target

Although `mpath-var-lib-containers.service` is set to only run on first
boot, it sometime runs twice when the system reboots too early.
Sometimes, in low load CI environement, the reboot in this test happens
before systemd's `first-boot-complete.target` is reached. This make
`ConditionFirstBoot` to still be true at the next boot, causing the
mpath service to fail, because it already ran during the actual first
boot.

A previous attempt[1] at fixing this improved the flake but this happened
again and i noticed that systemd didn't reach this target before the
reboot:
`Reached target first-boot-complete.target - First Boot Complete` is
only shown after the second boot in the logs.

Likely fixes coreos/rhel-coreos-config#66

[1] abd0c18

Author: jbtrystram

mantle/kola/tests/misc/multipath.go

@@ -15,14 +15,17 @@
 package misc
 
 import (
+	"fmt"
 	"strings"
+	"time"
 
 	coreosarch "github.com/coreos/stream-metadata-go/arch"
 
 	"github.com/coreos/coreos-assembler/mantle/kola/cluster"
 	"github.com/coreos/coreos-assembler/mantle/kola/register"
 	"github.com/coreos/coreos-assembler/mantle/platform"
 	"github.com/coreos/coreos-assembler/mantle/platform/conf"
+	"github.com/coreos/coreos-assembler/mantle/util"
 )
 
 var (
@@ -168,6 +171,8 @@ func verifyMultipath(c cluster.TestCluster, m platform.Machine, path string) {
 func runMultipathDay1(c cluster.TestCluster) {
 	m := c.Machines()[0]
 	verifyMultipathBoot(c, m)
+	// wait until first-boot-complete.target is reached
+	waitForCompleteFirstboot(c)
 	if err := m.Reboot(); err != nil {
 		c.Fatalf("Failed to reboot the machine: %v", err)
 	}
@@ -188,8 +193,30 @@ func runMultipathDay2(c cluster.TestCluster) {
 func runMultipathPartition(c cluster.TestCluster) {
 	m := c.Machines()[0]
 	verifyMultipath(c, m, "/var/lib/containers")
+	// wait until first-boot-complete.target is reached
+	waitForCompleteFirstboot(c)
 	if err := m.Reboot(); err != nil {
 		c.Fatalf("Failed to reboot the machine: %v", err)
 	}
 	verifyMultipath(c, m, "/var/lib/containers")
 }
+
+func waitForCompleteFirstboot(c cluster.TestCluster) {
+	m := c.Machines()[0]
+	err := util.WaitUntilReady(2*time.Minute, 10*time.Second, func() (bool, error) {
+
+		_, err := c.SSH(m, "sudo systemd-run --wait --quiet --property='After=first-boot-complete.target' echo 'firstboot complete'")
+		// get the actual target state
+		firstbootTargetState, err := c.SSH(m, "systemctl is-active first-boot-complete.target")
+
+		if err != nil {
+			return false, err
+		} else if string(firstbootTargetState) != "active" {
+			return false, fmt.Errorf("first-boot-complete.target state: %s.", string(firstbootTargetState))
+		}
+		return true, nil
+	})
+	if err != nil {
+		c.Fatalf("Timed out while waiting for first-boot-complete.target to be ready: %v", err)
+	}
+}