After stack rm, waits until all services are down (#835)

* After stack rm, waits until all services are down
asserts published ports are not -1

* Waits until container stops
Cleanup and added comments

Author: pcrespov

services/web/server/tests/integration/conftest.py

@@ -62,11 +62,13 @@ def webserver_environ(request, devel_environ, services_docker_compose, docker_st
     for name in core_services:
         if 'ports' not in services_docker_compose['services'][name]:
             continue
-        
+
         # published port is sometimes dynamically defined by the swarm
+        published_port = get_service_published_port(name)
+        assert published_port != "-1"
 
         environ['%s_HOST' % name.upper()] = '127.0.0.1'
-        environ['%s_PORT' % name.upper()] = get_service_published_port(name)
+        environ['%s_PORT' % name.upper()] = published_port
         # to swarm boundary since webserver is installed in the host and therefore outside the swarm's network
     from pprint import pprint
     pprint(environ)
@@ -121,9 +123,9 @@ def get_service_published_port(service_name: str) -> str:
     if not services:
         return published_port
     service_endpoint = services[0].attrs["Endpoint"]
-    
+
     if "Ports" not in service_endpoint or not service_endpoint["Ports"]:
         return published_port
-    
+
     published_port = service_endpoint["Ports"][0]["PublishedPort"]
-    return str(published_port)
+    return str(published_port)

services/web/server/tests/integration/fixtures/docker_registry.py

@@ -7,7 +7,7 @@
 import docker
 import pytest
 import tenacity
-
+import time
 
 @pytest.fixture(scope="session")
 def docker_registry():
@@ -46,6 +46,9 @@ def docker_registry():
 
     container.stop()
 
+    while docker_client.containers.list(filters={"name": container.name}):
+        time.sleep(1)
+
 @tenacity.retry(wait=tenacity.wait_fixed(1), stop=tenacity.stop_after_delay(60))
 def _wait_till_registry_is_responsive(url):
     docker_client = docker.from_env()

services/web/server/tests/integration/fixtures/docker_swarm.py

@@ -5,6 +5,7 @@
 # pylint:disable=redefined-outer-name
 
 import subprocess
+import time
 from pathlib import Path
 
 import docker
@@ -24,6 +25,7 @@ def docker_swarm(docker_client):
     # teardown
     assert docker_client.swarm.leave(force=True)
 
+
 @pytest.fixture(scope='module')
 def docker_stack(docker_swarm, docker_client, docker_compose_file: Path, tools_docker_compose_file: Path):
     docker_compose_ignore_file = docker_compose_file.parent / "docker-compose.ignore.yml"
@@ -44,10 +46,46 @@ def docker_stack(docker_swarm, docker_client, docker_compose_file: Path, tools_d
         )
     assert process.returncode == 0, "Error in '{}'".format(cmd)
 
+
     with docker_compose_ignore_file.open() as fp:
         docker_stack_cfg = yaml.safe_load(fp)
-        yield docker_stack_cfg
 
-    # clean up
+    def _print_services(msg):
+        from pprint import pprint
+        print("{:*^100}".format("docker services list " + msg))
+        for service in docker_client.services.list():
+            pprint(service.attrs)
+        print("-"*100)
+
+
+    _print_services("[BEFORE TEST]")
+
+    yield docker_stack_cfg
+
+    _print_services("[AFTER TEST]")
+
+    # clean up. Guarantees that all services are down before creating a new stack!
+    #
+    # WORKAROUND https://github.com/moby/moby/issues/30942#issue-207070098
+    #
+    # docker stack rm services
+
+    # until [ -z "$(docker service ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do
+    # sleep 1;
+    # done
+
+    # until [ -z "$(docker network ls --filter label=com.docker.stack.namespace=services -q)" ] || [ "$limit" -lt 0 ]; do
+    # sleep 1;
+    # done
+
     assert subprocess.run("docker stack rm services", shell=True).returncode == 0
+
+    while docker_client.services.list(filters={"label":"com.docker.stack.namespace=services"}):
+        time.sleep(1)
+
+    while docker_client.networks.list(filters={"label":"com.docker.stack.namespace=services"}):
+        time.sleep(1)
+
     docker_compose_ignore_file.unlink()
+
+    _print_services("[AFTER REMOVED]")