test: refactor so that all vm related tests can be reused

This commit reshuffles the code a bit so that we a reusable
./test/vmtest directory that can be used by the images library.

With that we can add a toplevel pyproject.toml file so that
we can import vmtest via
```console
$ pip install [email protected]/osbuild/bootc-image-builder
```
im other projects.

Note that none of this is ideal, butt this is (hopefully)
a temporary measure until we find a more permanent home for
our vm runner or replace it with something like test.thing or
the new osbuild QEMU code.

Author: mvo5

pyproject.toml

@@ -0,0 +1,14 @@
+# Note that this is pyproject file is here only for the vmtest utils.
+# This should move out eventually to its own repo or a different place
+# like "images".
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vmtest"
+version = "0.1.0"
+
+[tool.setuptools.packages.find]
+include = ["vmtest"]

test/conftest.py

@@ -1,6 +1,7 @@
 import pytest
 
 from testcases import TestCase
+from vmtest.util import get_free_port
 
 
 def pytest_addoption(parser):
@@ -20,3 +21,8 @@ def pytest_make_parametrize_id(config, val):  # pylint: disable=W0613
     if isinstance(val, TestCase):
         return f"{val}"
     return None
+
+
+@pytest.fixture(name="free_port")
+def free_port_fixture():
+    return get_free_port()

test/test_build_disk.py

@@ -18,7 +18,8 @@
 import testutil
 from containerbuild import build_container_fixture    # pylint: disable=unused-import
 from testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases
-from vm import AWS, QEMU
+import vmtest.util
+from vmtest.vm import AWS_REGION, AWS, QEMU
 
 if not testutil.has_executable("podman"):
     pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True)
@@ -113,7 +114,7 @@ def registry_conf_fixture(shared_tmpdir, request):
       {local_registry}:
         lookaside: file:///{sigstore_dir}
     """
-    registry_port = testutil.get_free_port()
+    registry_port = vmtest.util.get_free_port()
     # We cannot use localhost as we need to access the registry from both
     # the host system and the bootc-image-builder container.
     default_ip = testutil.get_ip_from_default_route()
@@ -410,7 +411,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_
 
                 upload_args = [
                     f"--aws-ami-name=bootc-image-builder-test-{str(uuid.uuid4())}",
-                    f"--aws-region={testutil.AWS_REGION}",
+                    f"--aws-region={AWS_REGION}",
                     "--aws-bucket=bootc-image-builder-ci",
                 ]
             elif force_aws_upload:
@@ -492,7 +493,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_
         metadata["ami_id"] = parse_ami_id_from_log(journal_output)
 
         def del_ami():
-            testutil.deregister_ami(metadata["ami_id"])
+            testutil.deregister_ami(metadata["ami_id"], AWS_REGION)
         request.addfinalizer(del_ami)
 
     journal_log_path.write_text(journal_output, encoding="utf8")

test/test_build_iso.py

@@ -8,7 +8,7 @@
 import testutil
 from containerbuild import build_container_fixture    # pylint: disable=unused-import
 from testcases import gen_testcases
-from vm import QEMU
+from vmtest.vm import QEMU
 
 from test_build_disk import (
     assert_kernel_args,

test/testutil.py

@@ -2,15 +2,11 @@
 import pathlib
 import platform
 import shutil
-import socket
 import subprocess
-import time
 
 import boto3
 from botocore.exceptions import ClientError
 
-AWS_REGION = "us-east-1"
-
 
 def run_journalctl(*args):
     pre = []
@@ -35,28 +31,6 @@ def has_executable(name):
     return shutil.which(name) is not None
 
 
-def get_free_port() -> int:
-    # this is racy but there is no race-free way to do better with the qemu CLI
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("localhost", 0))
-        return s.getsockname()[1]
-
-
-def wait_ssh_ready(address, port, sleep, max_wait_sec):
-    for _ in range(int(max_wait_sec / sleep)):
-        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            s.settimeout(sleep)
-            try:
-                s.connect((address, port))
-                data = s.recv(256)
-                if b"OpenSSH" in data:
-                    return
-            except (ConnectionRefusedError, ConnectionResetError, TimeoutError):
-                pass
-            time.sleep(sleep)
-    raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s")
-
-
 def has_x86_64_v3_cpu():
     # x86_64-v3 has multiple features, see
     # https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
@@ -95,8 +69,8 @@ def write_aws_creds(path):
     return True
 
 
-def deregister_ami(ami_id):
-    ec2 = boto3.resource("ec2", region_name=AWS_REGION)
+def deregister_ami(ami_id, aws_region):
+    ec2 = boto3.resource("ec2", region_name=aws_region)
     try:
         print(f"Deregistering image {ami_id}")
         ami = ec2.Image(ami_id)

vmtest/__init__.py

@@ -0,0 +1,24 @@
+import socket
+import time
+
+
+def get_free_port() -> int:
+    # this is racy but there is no race-free way to do better with the qemu CLI
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("localhost", 0))
+        return s.getsockname()[1]
+
+
+def wait_ssh_ready(address, port, sleep, max_wait_sec):
+    for _ in range(int(max_wait_sec / sleep)):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.settimeout(sleep)
+            try:
+                s.connect((address, port))
+                data = s.recv(256)
+                if b"OpenSSH" in data:
+                    return
+            except (ConnectionRefusedError, ConnectionResetError, TimeoutError):
+                pass
+            time.sleep(sleep)
+    raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s")

vmtest/util.py

@@ -1,31 +1,29 @@
 import contextlib
-import platform
+import shutil
 import subprocess
 from unittest.mock import call, patch
 
 import pytest
-from testutil import get_free_port, has_executable, wait_ssh_ready
+
+from vmtest.util import get_free_port, wait_ssh_ready
 
 
 def test_get_free_port():
     port_nr = get_free_port()
     assert 1024 < port_nr < 65535
 
 
-@pytest.fixture(name="free_port")
-def free_port_fixture():
-    return get_free_port()
-
-
 @patch("time.sleep")
-def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep, free_port):
+def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep):
+    free_port = get_free_port()
     with pytest.raises(ConnectionRefusedError):
         wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35)
     assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)]
 
 
-@pytest.mark.skipif(not has_executable("nc"), reason="needs nc")
-def test_wait_ssh_ready_sleeps_wrong_reply(free_port):
+@pytest.mark.skipif(not shutil.which("nc"), reason="needs nc")
+def test_wait_ssh_ready_sleeps_wrong_reply():
+    free_port = get_free_port()
     with contextlib.ExitStack() as cm:
         with subprocess.Popen(
             f"echo not-ssh | nc -vv -l -p {free_port}",
@@ -47,12 +45,3 @@ def test_wait_ssh_ready_sleeps_wrong_reply(free_port):
                     wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55)
                 assert mocked_sleep.call_args_list == [
                     call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)]
-
-
-@pytest.mark.skipif(platform.system() == "Darwin", reason="hangs on macOS")
-@pytest.mark.skipif(not has_executable("nc"), reason="needs nc")
-def test_wait_ssh_ready_integration(free_port):
-    with contextlib.ExitStack() as cm:
-        with subprocess.Popen(f"echo OpenSSH | nc -l -p {free_port}", shell=True) as p:
-            cm.callback(p.kill)
-            wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=10)

test/testutil_test.py renamed to vmtest/util_test.py

@@ -12,7 +12,9 @@
 import paramiko
 from botocore.exceptions import ClientError
 from paramiko.client import AutoAddPolicy, SSHClient
-from testutil import AWS_REGION, get_free_port, wait_ssh_ready
+from vmtest.util import get_free_port, wait_ssh_ready
+
+AWS_REGION = "us-east-1"
 
 
 class VM(abc.ABC):