test: refactor so that all vm related tests can be reused

pyproject.toml

@@ -0,0 +1,14 @@
+# Note that this is pyproject file is here only for the vmtest utils.
+# This should move out eventually to its own repo or a different place
+# like "images".
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vmtest"
+version = "0.1.0"
+
+[tool.setuptools.packages.find]
+include = ["vmtest"]

test/conftest.py

@@ -1,6 +1,8 @@
+from test.testcases import TestCase
+
 import pytest
 
-from testcases import TestCase
+from vmtest.util import get_free_port
 
 
 def pytest_addoption(parser):
@@ -20,3 +22,8 @@ def pytest_make_parametrize_id(config, val):  # pylint: disable=W0613
     if isinstance(val, TestCase):
         return f"{val}"
     return None
+
+
+@pytest.fixture(name="free_port")
+def free_port_fixture():
+    return get_free_port()

test/test_build_cross.py

@@ -1,18 +1,11 @@
 import platform
+from test.test_build_disk import (  # pylint: disable=unused-import
+    assert_disk_image_boots, build_container_fixture, gpg_conf_fixture,
+    image_type_fixture, registry_conf_fixture, shared_tmpdir_fixture)
+from test.testcases import gen_testcases
 
 import pytest
 
-from testcases import gen_testcases
-
-from test_build_disk import (  # pylint: disable=unused-import
-    assert_disk_image_boots,
-    build_container_fixture,
-    gpg_conf_fixture,
-    image_type_fixture,
-    registry_conf_fixture,
-    shared_tmpdir_fixture,
-)
-
 
 # This testcase is not part of "test_build_disk.py:test_image_boots"
 # because it takes ~30min on the GH runners so moving it into a

test/test_build_disk.py

@@ -9,16 +9,20 @@
 import subprocess
 import tempfile
 import uuid
-from contextlib import contextmanager, ExitStack
-from typing import NamedTuple
+from contextlib import ExitStack, contextmanager
 from dataclasses import dataclass
+# local test utils
+from test import testutil
+from typing import NamedTuple
 
 import pytest
-# local test utils
-import testutil
-from containerbuild import build_container_fixture    # pylint: disable=unused-import
-from testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases
-from vm import AWS, QEMU
+
+import vmtest.util
+from vmtest.vm import AWS, AWS_REGION, QEMU
+
+from .containerbuild import \
+    build_container_fixture  # pylint: disable=unused-import
+from .testcases import CLOUD_BOOT_IMAGE_TYPES, DISK_IMAGE_TYPES, gen_testcases
 
 if not testutil.has_executable("podman"):
     pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True)
@@ -113,7 +117,7 @@ def registry_conf_fixture(shared_tmpdir, request):
       {local_registry}:
         lookaside: file:///{sigstore_dir}
     """
-    registry_port = testutil.get_free_port()
+    registry_port = vmtest.util.get_free_port()
     # We cannot use localhost as we need to access the registry from both
     # the host system and the bootc-image-builder container.
     default_ip = testutil.get_ip_from_default_route()
@@ -410,7 +414,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_
 
                 upload_args = [
                     f"--aws-ami-name=bootc-image-builder-test-{str(uuid.uuid4())}",
-                    f"--aws-region={testutil.AWS_REGION}",
+                    f"--aws-region={AWS_REGION}",
                     "--aws-bucket=bootc-image-builder-ci",
                 ]
             elif force_aws_upload:
@@ -492,7 +496,7 @@ def build_images(shared_tmpdir, build_container, request, force_aws_upload, gpg_
         metadata["ami_id"] = parse_ami_id_from_log(journal_output)
 
         def del_ami():
-            testutil.deregister_ami(metadata["ami_id"])
+            testutil.deregister_ami(metadata["ami_id"], AWS_REGION)
         request.addfinalizer(del_ami)
 
     journal_log_path.write_text(journal_output, encoding="utf8")

test/test_build_iso.py

@@ -2,24 +2,19 @@
 import platform
 import subprocess
 from contextlib import ExitStack
+# local test utils
+from test import testutil
+from test.test_build_disk import (  # pylint: disable=unused-import
+    ImageBuildResult, assert_kernel_args, gpg_conf_fixture, image_type_fixture,
+    registry_conf_fixture, shared_tmpdir_fixture)
+from test.testcases import gen_testcases
 
 import pytest
-# local test utils
-import testutil
-from containerbuild import build_container_fixture    # pylint: disable=unused-import
-from testcases import gen_testcases
-from vm import QEMU
 
-from test_build_disk import (
-    assert_kernel_args,
-    ImageBuildResult,
-)
-from test_build_disk import (  # pylint: disable=unused-import
-    gpg_conf_fixture,
-    image_type_fixture,
-    registry_conf_fixture,
-    shared_tmpdir_fixture,
-)
+from vmtest.vm import QEMU
+
+from .containerbuild import \
+    build_container_fixture  # pylint: disable=unused-import
 
 
 @pytest.mark.skipif(platform.system() != "Linux", reason="boot test only runs on linux right now")

test/test_manifest.py

@@ -7,14 +7,13 @@
 import platform
 import subprocess
 import textwrap
+from test import testutil
+from test.containerbuild import build_container_fixture as _
+from test.containerbuild import make_container
+from test.testcases import gen_testcases
 
 import pytest
 
-import testutil
-from containerbuild import build_container_fixture as _
-from containerbuild import make_container
-from testcases import gen_testcases
-
 if not testutil.has_executable("podman"):
     pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True)
 if not testutil.can_start_rootful_containers():

test/test_opts.py

@@ -1,11 +1,12 @@
 import os
 import platform
 import subprocess
+from test import testutil
+# pylint: disable=unused-import
+from test.containerbuild import (build_container_fixture,
+                                 build_fake_container_fixture)
 
 import pytest
-import testutil
-# pylint: disable=unused-import
-from containerbuild import build_container_fixture, build_fake_container_fixture
 
 
 @pytest.fixture(name="container_storage", scope="session")

test/test_progress.py

@@ -1,15 +1,14 @@
 import subprocess
-
-import pytest
-
-import testutil
-# pylint: disable=unused-import,duplicate-code
-from test_opts import container_storage_fixture
-from containerbuild import (
+from test import testutil
+from test.containerbuild import (  # pylint: disable=unused-import
     build_container_fixture,
     build_erroring_container_fixture,
     build_fake_container_fixture,
 )
+# pylint: disable=unused-import,duplicate-code
+from test.test_opts import container_storage_fixture
+
+import pytest
 
 
 def test_progress_debug(tmp_path, build_fake_container):

test/testutil.py

@@ -2,15 +2,11 @@
 import pathlib
 import platform
 import shutil
-import socket
 import subprocess
-import time
 
 import boto3
 from botocore.exceptions import ClientError
 
-AWS_REGION = "us-east-1"
-
 
 def run_journalctl(*args):
     pre = []
@@ -35,28 +31,6 @@ def has_executable(name):
     return shutil.which(name) is not None
 
 
-def get_free_port() -> int:
-    # this is racy but there is no race-free way to do better with the qemu CLI
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("localhost", 0))
-        return s.getsockname()[1]
-
-
-def wait_ssh_ready(address, port, sleep, max_wait_sec):
-    for _ in range(int(max_wait_sec / sleep)):
-        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-            s.settimeout(sleep)
-            try:
-                s.connect((address, port))
-                data = s.recv(256)
-                if b"OpenSSH" in data:
-                    return
-            except (ConnectionRefusedError, ConnectionResetError, TimeoutError):
-                pass
-            time.sleep(sleep)
-    raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s")
-
-
 def has_x86_64_v3_cpu():
     # x86_64-v3 has multiple features, see
     # https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
@@ -95,8 +69,8 @@ def write_aws_creds(path):
     return True
 
 
-def deregister_ami(ami_id):
-    ec2 = boto3.resource("ec2", region_name=AWS_REGION)
+def deregister_ami(ami_id, aws_region):
+    ec2 = boto3.resource("ec2", region_name=aws_region)
     try:
         print(f"Deregistering image {ami_id}")
         ami = ec2.Image(ami_id)

vmtest/util.py

@@ -0,0 +1,24 @@
+import socket
+import time
+
+
+def get_free_port() -> int:
+    # this is racy but there is no race-free way to do better with the qemu CLI
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("localhost", 0))
+        return s.getsockname()[1]
+
+
+def wait_ssh_ready(address, port, sleep, max_wait_sec):
+    for _ in range(int(max_wait_sec / sleep)):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.settimeout(sleep)
+            try:
+                s.connect((address, port))
+                data = s.recv(256)
+                if b"OpenSSH" in data:
+                    return
+            except (ConnectionRefusedError, ConnectionResetError, TimeoutError):
+                pass
+            time.sleep(sleep)
+    raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s")

test/testutil_test.py → vmtest/util_test.py

@@ -1,31 +1,29 @@
 import contextlib
-import platform
+import shutil
 import subprocess
 from unittest.mock import call, patch
 
 import pytest
-from testutil import get_free_port, has_executable, wait_ssh_ready
+
+from vmtest.util import get_free_port, wait_ssh_ready
 
 
 def test_get_free_port():
     port_nr = get_free_port()
     assert 1024 < port_nr < 65535
 
 
-@pytest.fixture(name="free_port")
-def free_port_fixture():
-    return get_free_port()
-
-
 @patch("time.sleep")
-def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep, free_port):
+def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep):
+    free_port = get_free_port()
     with pytest.raises(ConnectionRefusedError):
         wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.35)
     assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)]
 
 
-@pytest.mark.skipif(not has_executable("nc"), reason="needs nc")
-def test_wait_ssh_ready_sleeps_wrong_reply(free_port):
+@pytest.mark.skipif(not shutil.which("nc"), reason="needs nc")
+def test_wait_ssh_ready_sleeps_wrong_reply():
+    free_port = get_free_port()
     with contextlib.ExitStack() as cm:
         with subprocess.Popen(
             f"echo not-ssh | nc -vv -l -p {free_port}",
@@ -47,12 +45,3 @@ def test_wait_ssh_ready_sleeps_wrong_reply(free_port):
                     wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=0.55)
                 assert mocked_sleep.call_args_list == [
                     call(0.1), call(0.1), call(0.1), call(0.1), call(0.1)]
-
-
-@pytest.mark.skipif(platform.system() == "Darwin", reason="hangs on macOS")
-@pytest.mark.skipif(not has_executable("nc"), reason="needs nc")
-def test_wait_ssh_ready_integration(free_port):
-    with contextlib.ExitStack() as cm:
-        with subprocess.Popen(f"echo OpenSSH | nc -l -p {free_port}", shell=True) as p:
-            cm.callback(p.kill)
-            wait_ssh_ready("localhost", free_port, sleep=0.1, max_wait_sec=10)

test/vm.py → vmtest/vm.py

@@ -12,7 +12,10 @@
 import paramiko
 from botocore.exceptions import ClientError
 from paramiko.client import AutoAddPolicy, SSHClient
-from testutil import AWS_REGION, get_free_port, wait_ssh_ready
+
+from vmtest.util import get_free_port, wait_ssh_ready
+
+AWS_REGION = "us-east-1"
 
 
 class VM(abc.ABC):