diff --git a/src/vmm/src/devices/virtio/block/virtio/device.rs b/src/vmm/src/devices/virtio/block/virtio/device.rs index bdd169ff171..b5a2bce3972 100644 --- a/src/vmm/src/devices/virtio/block/virtio/device.rs +++ b/src/vmm/src/devices/virtio/block/virtio/device.rs @@ -12,6 +12,7 @@ use std::io::{Seek, SeekFrom}; use std::os::linux::fs::MetadataExt; use std::path::PathBuf; use std::sync::Arc; +use std::time::Duration; use block_io::FileEngine; use serde::{Deserialize, Serialize}; @@ -423,6 +424,11 @@ impl VirtioBlock { } used_any = true; + if self.id == "scratch" + && (request.r#type == RequestType::In || request.r#type == RequestType::Out) + { + std::thread::sleep(Duration::from_millis(60)); + } request.process(&mut self.disk, head.index, mem, &self.metrics) } Err(err) => { diff --git a/src/vmm/src/devices/virtio/block/virtio/event_handler.rs b/src/vmm/src/devices/virtio/block/virtio/event_handler.rs index db69e23d7f0..129b05f9089 100644 --- a/src/vmm/src/devices/virtio/block/virtio/event_handler.rs +++ b/src/vmm/src/devices/virtio/block/virtio/event_handler.rs @@ -1,6 +1,7 @@ // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 use event_manager::{EventOps, Events, MutEventSubscriber}; +use log::info; use vmm_sys_util::epoll::EventSet; use super::io::FileEngine; @@ -85,7 +86,15 @@ impl MutEventSubscriber for VirtioBlock { if self.is_activated() { match source { Self::PROCESS_ACTIVATE => self.process_activate_event(ops), - Self::PROCESS_QUEUE => self.process_queue_event(), + Self::PROCESS_QUEUE => { + let tstamp = std::time::Instant::now(); + self.process_queue_event(); + info!( + "block[{}]: processed queue for {} usec", + &self.id, + tstamp.elapsed().as_micros() + ); + } Self::PROCESS_RATE_LIMITER => self.process_rate_limiter_event(), Self::PROCESS_ASYNC_COMPLETION => self.process_async_completion_event(), _ => warn!("Block: Spurious event received: {:?}", source), diff --git a/src/vmm/src/devices/virtio/net/event_handler.rs b/src/vmm/src/devices/virtio/net/event_handler.rs index 9d8c09a45f2..c4635086602 100644 --- a/src/vmm/src/devices/virtio/net/event_handler.rs +++ b/src/vmm/src/devices/virtio/net/event_handler.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use event_manager::{EventOps, Events, MutEventSubscriber}; +use log::info; use vmm_sys_util::epoll::EventSet; use crate::devices::virtio::device::VirtioDevice; @@ -97,6 +98,7 @@ impl MutEventSubscriber for Net { } if self.is_activated() { + let tstamp = std::time::Instant::now(); match source { Self::PROCESS_ACTIVATE => self.process_activate_event(ops), Self::PROCESS_VIRTQ_RX => self.process_rx_queue_event(), @@ -109,6 +111,10 @@ impl MutEventSubscriber for Net { self.metrics.event_fails.inc(); } } + info!( + "net: processed queue for {} usec", + tstamp.elapsed().as_micros() + ); } else { warn!( "Net: The device is not yet activated. Spurious event received: {:?}", diff --git a/tests/integration_tests/functional/test_drive_virtio.py b/tests/integration_tests/functional/test_drive_virtio.py index 9c61ead56a9..a1901cd12fd 100644 --- a/tests/integration_tests/functional/test_drive_virtio.py +++ b/tests/integration_tests/functional/test_drive_virtio.py @@ -2,13 +2,21 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for guest-side operations on /drives resources.""" +import concurrent.futures import os +import time import pytest import host_tools.drive as drive_tools from framework import utils from framework.utils_drive import partuuid_and_disk_path +from integration_tests.performance.test_block_ab import ( + BLOCK_DEVICE_SIZE_MB, + RUNTIME_SEC, + WARMUP_SEC, + prepare_microvm_for_test, +) MB = 1024 * 1024 @@ -383,3 +391,91 @@ def _check_mount(ssh_connection, dev_path): assert stderr == "" _, _, stderr = ssh_connection.run("umount /tmp", timeout=30.0) assert stderr == "" + + +def run_fio(microvm, mode, block_size, test_output_dir, fio_engine="libaio"): + """Run a fio test in the specified mode with block size bs.""" + cmd = ( + utils.CmdBuilder("fio") + .with_arg(f"--name={mode}-{block_size}") + .with_arg(f"--numjobs={microvm.vcpus_count}") + .with_arg(f"--runtime={RUNTIME_SEC}") + .with_arg("--time_based=1") + .with_arg(f"--ramp_time={WARMUP_SEC}") + .with_arg("--filename=/dev/vdb") + .with_arg("--direct=1") + .with_arg(f"--rw={mode}") + .with_arg("--randrepeat=0") + .with_arg(f"--bs={block_size}") + .with_arg(f"--size={BLOCK_DEVICE_SIZE_MB}M") + .with_arg(f"--ioengine={fio_engine}") + .with_arg("--iodepth=256") + # Set affinity of the entire fio process to a set of vCPUs equal in size to number of workers + .with_arg( + f"--cpus_allowed={','.join(str(i) for i in range(microvm.vcpus_count))}" + ) + # Instruct fio to pin one worker per vcpu + .with_arg("--cpus_allowed_policy=split") + .with_arg("--log_avg_msec=1000") + .with_arg(f"--write_bw_log={mode}") + .with_arg("--output-format=json+") + .with_arg("--output=/tmp/fio.json") + ) + + # Latency measurements only make sense for psync engine + if fio_engine == "psync": + cmd = cmd.with_arg(f"--write_lat_log={mode}") + + cmd = cmd.build() + + prepare_microvm_for_test(microvm) + + with concurrent.futures.ThreadPoolExecutor() as executor: + fio_future = executor.submit(_run_fio, microvm, cmd, test_output_dir) + while not fio_future.done(): + microvm.ssh.check_output("true", timeout=1) + fio_future.result() + + +def _run_fio(microvm, cmd, test_output_dir): + rc, stdout, stderr = microvm.ssh.run(f"cd /tmp; {cmd}") + assert rc == 0 + assert stderr == "" + print(f"standard output: {stdout}") + + microvm.ssh.scp_get("/tmp/fio.json", test_output_dir) + microvm.ssh.scp_get("/tmp/*.log", test_output_dir) + + +@pytest.mark.parametrize("vcpus", [1, 2], ids=["1vcpu", "2vcpu"]) +@pytest.mark.parametrize("fio_mode", ["randread", "randwrite"]) +@pytest.mark.parametrize("fio_block_size", [4096], ids=["bs4096"]) +@pytest.mark.parametrize("fio_engine", ["libaio", "psync"]) +def test_greedy_block( + microvm_factory, + guest_kernel_acpi, + rootfs, + vcpus, + fio_mode, + fio_block_size, + fio_engine, + io_engine, + results_dir, +): + """ + Make sure that a guest continuously using the block device + doesn't starve a Network device + """ + vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False) + vm.jailer.extra_args.update({"no-seccomp": None}) + vm.spawn(log_level="Info", emit_metrics=False) + vm.basic_config(vcpu_count=vcpus, mem_size_mib=1024) + vm.add_net_iface() + + # Add a secondary block device for testing + fs = drive_tools.FilesystemFile(os.path.join(vm.fsfiles, "scratch"), 4096) + vm.add_drive("scratch2", fs.path, io_engine=io_engine) + + vm.start() + + run_fio(vm, fio_mode, fio_block_size, results_dir, fio_engine) diff --git a/tests/integration_tests/performance/test_block_ab.py b/tests/integration_tests/performance/test_block_ab.py index dfd0728084a..6bf1d24ba48 100644 --- a/tests/integration_tests/performance/test_block_ab.py +++ b/tests/integration_tests/performance/test_block_ab.py @@ -73,7 +73,7 @@ def run_fio(microvm, mode, block_size, test_output_dir, fio_engine="libaio"): .with_arg("--output=/tmp/fio.json") ) - # Latency measurements only make sence for psync engine + # Latency measurements only make sense for psync engine if fio_engine == "psync": cmd = cmd.with_arg(f"--write_lat_log={mode}")