test: add post-restore latency test

Add a snapshot latency test that output metrics on how long it takes to
memset a 128MB memory region post-snapshot restore.

We only collect these latencies for a single configuration (1024MB,
2vCPU), since they will be the same no matter the size of the memory.

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/integration_tests/performance/test_snapshot_ab.py

@@ -1,7 +1,9 @@
 # Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 """Performance benchmark for snapshot restore."""
+import signal
 import tempfile
+import time
 from dataclasses import dataclass
 from functools import lru_cache
 
@@ -11,6 +13,7 @@
 from framework.microvm import Microvm
 
 USEC_IN_MSEC = 1000
+NS_IN_MSEC = 1_000_000
 ITERATIONS = 30
 
 
@@ -88,7 +91,7 @@ def configure_vm(self, microvm_factory, guest_kernel, rootfs, metrics) -> Microv
     "test_setup",
     [
         SnapshotRestoreTest(mem=128, vcpus=1),
-        SnapshotRestoreTest(mem=1024, vcpus=1),
+        SnapshotRestoreTest(mem=1024, vcpus=2),
         SnapshotRestoreTest(mem=2048, vcpus=2),
         SnapshotRestoreTest(mem=4096, vcpus=3),
         SnapshotRestoreTest(mem=6144, vcpus=4),
@@ -127,3 +130,38 @@ def test_restore_latency(
                 break
         assert value > 0
         metrics.put_metric("latency", value, "Milliseconds")
+
+
+@pytest.mark.nonci
+@pytest.mark.parametrize("uffd_handler", [None, "valid"])
+def test_post_restore_latency(
+    microvm_factory, rootfs, guest_kernel_linux_5_10, metrics, uffd_handler
+):
+    """Collects latency metric of post-restore memory accesses done inside the guest"""
+    test_setup = SnapshotRestoreTest(mem=1024, vcpus=2)
+    vm = test_setup.configure_vm(
+        microvm_factory, guest_kernel_linux_5_10, rootfs, metrics
+    )
+
+    vm.ssh.check_output(
+        "nohup /usr/local/bin/fast_page_fault_helper >/dev/null 2>&1 </dev/null &"
+    )
+
+    # Give helper time to initialize
+    time.sleep(5)
+
+    snapshot = vm.snapshot_full()
+    vm.kill()
+
+    for microvm in microvm_factory.build_n_from_snapshot(
+        snapshot, ITERATIONS, uffd_handler_name=uffd_handler
+    ):
+        _, pid, _ = microvm.ssh.check_output("pidof fast_page_fault_helper")
+
+        microvm.ssh.check_output(f"kill -s {signal.SIGUSR1} {pid}")
+
+        _, duration, _ = microvm.ssh.check_output(
+            "while [ ! -f /tmp/fast_page_fault_helper.out ]; do sleep 1; done; cat /tmp/fast_page_fault_helper.out"
+        )
+
+        metrics.put_metric("fault_latency", int(duration) / NS_IN_MSEC, "Milliseconds")