chore(ci): e2e lxd test preflight failure - wait for clock sync (#2623)

* chore(ci): e2e lxd test preflight failure - wait for clock sync

* f

* f

* f

* f

Author: emosbaugh

e2e/cluster/lxd/cluster.go

@@ -235,15 +235,16 @@ func NewCluster(in *ClusterInput) *Cluster {
 
 	wg := sync.WaitGroup{}
 	wg.Add(len(out.Nodes))
-	for _, node := range out.Nodes {
-		go func(node string) {
+	for i, node := range out.Nodes {
+		go func(i int, node string) {
 			defer wg.Done()
 			CopyFilesToNode(in, node)
 			CopyDirsToNode(in, node)
 			if in.CreateRegularUser {
 				CreateRegularUser(in, node)
 			}
-		}(node)
+			out.waitForClockSync(i)
+		}(i, node)
 	}
 	wg.Wait()
 
@@ -1136,6 +1137,23 @@ func (c *Cluster) Cleanup(envs ...map[string]string) {
 	c.copyPlaywrightReport()
 }
 
+func (c *Cluster) waitForClockSync(node int) {
+	timeout := time.After(5 * time.Minute)
+	tick := time.Tick(time.Second)
+	for {
+		select {
+		case <-timeout:
+			stdout, stderr, err := c.RunCommandOnNode(node, []string{"timedatectl", "show", "-p", "NTP", "-p", "CanNTP", "-p", "NTPSynchronized"})
+			c.T.Fatalf("timeout waiting for clock sync on node %d: %v: %s: %s", node, err, stdout, stderr)
+		case <-tick:
+			status, _, _ := c.RunCommandOnNode(node, []string{"timedatectl", "show", "-p", "NTPSynchronized"})
+			if strings.Contains(status, "NTPSynchronized=yes") {
+				return
+			}
+		}
+	}
+}
+
 func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (string, string, error) {
 	if err := c.SetupPlaywright(); err != nil {
 		return "", "", fmt.Errorf("failed to setup playwright: %w", err)