test: change the echo server in test_5_snapshots

With AL2023 and with a 4.14 kernel and some machine configurations, this
test can fail due to the vsock clients timing out during connect.

The problem disappears if we run socat instead of the internal echo
server that we run in a thread with Python. It may be the echo server is
buggy or the Python GIL is acting up.

So, replace the single-threaded echo server with socat, which can fork
per client and is an external process.

Signed-off-by: Pablo Barbáchano <[email protected]>

Author: pb8o

tests/framework/utils_vsock.py

@@ -5,11 +5,11 @@
 import hashlib
 import os.path
 import re
-from select import select
+import time
+from pathlib import Path
 from socket import AF_UNIX, SOCK_STREAM, socket
-from threading import Event, Thread
-
-from framework import utils
+from subprocess import Popen
+from threading import Thread
 
 ECHO_SERVER_PORT = 5252
 SERVER_ACCEPT_BACKLOG = 128
@@ -19,78 +19,6 @@
 VSOCK_UDS_PATH = "v.sock"
 
 
-class HostEchoServer(Thread):
-    """A simple "echo" server for vsock.
-
-    This server will accept incoming connections (initiated by the guest vm),
-    and, for each connection, it will read any incoming data and then echo it
-    right back.
-    """
-
-    def __init__(self, vm, path):
-        """."""
-        super().__init__()
-        self.vm = vm
-        self.path = path
-        self.sock = socket(AF_UNIX, SOCK_STREAM)
-        self.sock.bind(path)
-        self.sock.listen(SERVER_ACCEPT_BACKLOG)
-        self.error = None
-        self.clients = []
-        self.exit_evt = Event()
-
-        # Link the listening Unix socket into the VM's jail, so that
-        # Firecracker can connect to it.
-        vm.create_jailed_resource(path)
-
-    def run(self):
-        """Thread code payload.
-
-        Wrap up the real "run" into a catch-all block, because Python cannot
-        into threads - if this thread were to raise an unhandled exception,
-        the whole process would lock.
-        """
-        try:
-            self._run()
-        # pylint: disable=broad-except
-        except Exception as err:
-            self.error = err
-
-    def _run(self):
-        while not self.exit_evt.is_set():
-            watch_list = self.clients + [self.sock]
-            rd_list, _, _ = select(watch_list, [], [], 1)
-            for rdo in rd_list:
-                if rdo == self.sock:
-                    # Read event on the listening socket: a new client
-                    # wants to connect.
-                    (client, _) = self.sock.accept()
-                    self.clients.append(client)
-                    continue
-                # Read event on a connected socket: new data is
-                # available from some client.
-                buf = rdo.recv(BUF_SIZE)
-                if not buf:
-                    # Zero-length read: connection reset by peer.
-                    self.clients.remove(rdo)
-                    continue
-                sent = 0
-                while sent < len(buf):
-                    # Send back everything we just read.
-                    sent += rdo.send(buf[sent:])
-
-    def exit(self):
-        """Shut down the echo server and wait for it to exit.
-
-        This method can be called from any thread. Upon returning, the
-        echo server will have shut down.
-        """
-        self.exit_evt.set()
-        self.join()
-        self.sock.close()
-        utils.run_cmd("rm -f {}".format(self.path))
-
-
 class HostEchoWorker(Thread):
     """A vsock echo worker, connecting to a guest echo server.
 
@@ -199,8 +127,19 @@ def check_guest_connections(vm, server_port_path, blob_path, blob_hash):
     start `TEST_CONNECTION_COUNT` workers inside the guest VM, all
     communicating with the echo server.
     """
-    echo_server = HostEchoServer(vm, server_port_path)
-    echo_server.start()
+
+    echo_server = Popen(
+        ["socat", f"UNIX-LISTEN:{server_port_path},fork,backlog=5", "exec:'/bin/cat'"]
+    )
+
+    # Link the listening Unix socket into the VM's jail, so that
+    # Firecracker can connect to it.
+    attempt = 0
+    # But 1st, give socat a bit of time to create the socket
+    while not Path(server_port_path).exists() and attempt < 3:
+        time.sleep(0.2)
+        attempt += 1
+    vm.create_jailed_resource(server_port_path)
 
     # Increase maximum process count for the ssh service.
     # Avoids: "bash: fork: retry: Resource temporarily unavailable"
@@ -236,9 +175,10 @@ def check_guest_connections(vm, server_port_path, blob_path, blob_hash):
     cmd += "for w in $workers; do wait $w || exit -1; done"
 
     ecode, stdout, stderr = vm.ssh.run(cmd)
-
-    echo_server.exit()
-    assert echo_server.error is None
+    echo_server.terminate()
+    rc = echo_server.wait()
+    # socat exits with 128 + 15 (SIGTERM)
+    assert rc == 143
 
     print(stdout.read())
     assert ecode == 0, stderr.read()

tools/devctr/Dockerfile

@@ -41,7 +41,8 @@ RUN apt-get update \
         libcurl4-openssl-dev \
         lsof \
         musl-tools \
-        net-tools iproute2 iperf3 fdisk \
+        # needed for integration tests
+        net-tools iproute2 iperf3 socat fdisk \
         openssh-client \
         pkgconf \
         python3 python3-dev python3-pip \

tools/devtool

@@ -72,7 +72,7 @@
 DEVCTR_IMAGE_NO_TAG="public.ecr.aws/firecracker/fcuvm"
 
 # Development container tag
-DEVCTR_IMAGE_TAG=${DEVCTR_IMAGE_TAG:-v59}
+DEVCTR_IMAGE_TAG=${DEVCTR_IMAGE_TAG:-v60}
 
 # Development container image (name:tag)
 # This should be updated whenever we upgrade the development container.