move the random write to a separate systemd unit

Instead of calling c.ssh() in the goroutine
move the code to a systemd service and enable that
Maybe it's worth adding another `util.Retry`
to wait for the service to complete before
continuing ?

Author: jbtrystram

mantle/kola/tests/ostree/sync.go

@@ -157,7 +157,35 @@ storage:
       contents:
         inline: |
           [Manager]
-          DefaultTimeoutStopSec=5s`)
+          DefaultTimeoutStopSec=5s
+    - path: /usr/local/bin/nfs-random-write.sh
+      mode: 0755
+      overwrite: true
+      contents:
+        inline: |
+          #!/bin/bash
+          for i in $(seq 6); do
+            (while sudo rm -f /var/tmp/data$i/test; do \
+               for x in $(seq 6); do \
+                 set -x; \
+                 sudo dd if=/dev/urandom of=/var/tmp/data$i/test bs=4096 count=2048 conv=notrunc oflag=append &> /dev/null; \
+                 set +x; \
+                 sleep 0.5; \
+                done; \
+             done) &
+           done
+systemd:
+  units:
+  - name: write-to-nfs.service
+    enabled: false
+    contents: |
+      [Unit]
+      ConditionFirstBoot=true
+      [Service]
+      Type=oneshot
+      RemainAfterExit=yes
+      ExecStart=/usr/local/bin/nfs-random-write.sh
+      Restart=on-failure`)
 	opts := platform.MachineOptions{
 		MinMemory: 2048,
 	}
@@ -197,21 +225,10 @@ storage:
 func doSyncTest(c cluster.TestCluster, client platform.Machine) {
 	c.RunCmdSync(client, "sudo touch /var/tmp/data3/test")
 	// Continue write
-	go func() {
-		_, err := c.SSH(client, `for i in $(seq 6); do
-			(while sudo rm -f /var/tmp/data$i/test; do \
-				for x in $(seq 6); do \
-					set -x; \
-					sudo dd if=/dev/urandom of=/var/tmp/data$i/test bs=4096 count=2048 conv=notrunc oflag=append &> /dev/null; \
-					set +x; \
-					sleep 0.5; \
-				done; \
-			done) &
-		done`)
-		if err != nil {
-			c.Fatalf("failed to run dd command: %v", err)
-		}
-	}()
+	_, err := c.SSH(client, `sudo systemctl enable --now write-to-nfs.service`)
+	if err != nil {
+		c.Fatalf("failed to start service: %v", err)
+	}
 
 	// Create a stage deploy using kargs while writing
 	c.RunCmdSync(client, "sudo rpm-ostree kargs --append=test=1")
@@ -224,7 +241,7 @@ func doSyncTest(c cluster.TestCluster, client platform.Machine) {
 	c.Log("Set link down and rebooting.")
 	// Skip the error check as it is expected
 	cmd := fmt.Sprintf("sudo systemd-run sh -c 'ip link set %s down && sleep 5 && systemctl reboot'", netdevice)
-	_, err := c.SSH(client, cmd)
+	_, err = c.SSH(client, cmd)
 	if err != nil {
 		c.Fatalf("failed to set down link and reboot: %v", err)
 	}