Merge pull request pimutils#897 from pimutils/optimise-test-servers

Run test servers as fixtures

Author: Hugo Osvaldo Barrera

.builds/archlinux.yaml

@@ -24,19 +24,16 @@ environment:
   BUILD: test
   CI: true
   CODECOV_TOKEN: b834a3c5-28fa-4808-9bdb-182210069c79
+  DAV_SERVER: radicale xandikos
   REQUIREMENTS: release
   # TODO: ETESYNC_TESTS
 tasks:
   - setup: |
+      sudo systemctl start docker
       cd vdirsyncer
       python setup.py build
       sudo pip install --no-index .
-      sudo systemctl start docker
-      DAV_SERVER="radicale xandikos" make -e install-servers
   - test: |
       cd vdirsyncer
-      # Non-system python is used for packages:
-      export PATH=$PATH:~/.local/bin/
       make -e ci-test
-      DAV_SERVER=radicale make -e ci-test-storage
-      DAV_SERVER=xandikos make -e ci-test-storage
+      make -e ci-test-storage

.builds/tests-minimal.yaml

@@ -14,17 +14,17 @@ environment:
   BUILD: test
   CI: true
   CODECOV_TOKEN: b834a3c5-28fa-4808-9bdb-182210069c79
+  DAV_SERVER: radicale xandikos
   REQUIREMENTS: minimal
   # TODO: ETESYNC_TESTS
 tasks:
   - setup: |
-      cd vdirsyncer
       sudo systemctl start docker
-      DAV_SERVER="radicale xandikos" make -e install-test
+      cd vdirsyncer
+      make -e install-dev
   - test: |
       cd vdirsyncer
       # Non-system python is used for packages:
       export PATH=$PATH:~/.local/bin/
       make -e ci-test
-      DAV_SERVER=radicale make -e ci-test-storage
-      DAV_SERVER=xandikos make -e ci-test-storage
+      make -e ci-test-storage

.builds/tests-release.yaml

@@ -11,22 +11,20 @@ environment:
   BUILD: test
   CI: true
   CODECOV_TOKEN: b834a3c5-28fa-4808-9bdb-182210069c79
+  DAV_SERVER: baikal radicale xandikos
   REQUIREMENTS: release
   # TODO: ETESYNC_TESTS
 tasks:
   - setup: |
-      cd vdirsyncer
       sudo systemctl start docker
-      DAV_SERVER="radicale xandikos baikal" make -e install-test
-      make -e install-style
+      cd vdirsyncer
+      make -e install-dev -e install-docs
   - test: |
       cd vdirsyncer
       # Non-system python is used for packages:
       export PATH=$PATH:~/.local/bin/
       make -e ci-test
-      DAV_SERVER=baikal make -e ci-test-storage
-      DAV_SERVER=radicale make -e ci-test-storage
-      DAV_SERVER=xandikos make -e ci-test-storage
+      make -e ci-test-storage
   - style: |
       cd vdirsyncer
       # Non-system python is used for packages:

Makefile

@@ -36,10 +36,11 @@ ifeq ($(ETESYNC_TESTS), true)
 endif
 
 PYTEST = py.test $(PYTEST_ARGS)
-
-export TESTSERVER_BASE := ./tests/storage/servers/
 CODECOV_PATH = /tmp/codecov.sh
 
+all:
+	$(error Take a look at https://vdirsyncer.pimutils.org/en/stable/tutorial.html#installation)
+
 ci-test:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
 	$(PYTEST) tests/unit/
@@ -50,35 +51,14 @@ ci-test:
 
 ci-test-storage:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
-	$(PYTEST) tests/storage/
-	bash $(CODECOV_PATH) -c -F storage
-
-test:
-	$(PYTEST)
-
-all:
-	$(error Take a look at https://vdirsyncer.pimutils.org/en/stable/tutorial.html#installation)
-
-install-servers:
 	set -ex; \
 	for server in $(DAV_SERVER); do \
-		if [ -f $(TESTSERVER_BASE)$$server/install.sh ]; then \
-			(cd $(TESTSERVER_BASE)$$server && sh install.sh); \
-		fi \
+		DAV_SERVER=$$server $(PYTEST) --cov-append tests/storage; \
 	done
+	bash $(CODECOV_PATH) -c -F storage
 
-install-test: install-servers install-dev
-	pip install -Ur test-requirements.txt
-	set -xe && if [ "$$REQUIREMENTS" = "devel" ]; then \
-		pip install -U --force-reinstall \
-			git+https://github.com/DRMacIver/hypothesis \
-			git+https://github.com/kennethreitz/requests \
-			git+https://github.com/pytest-dev/pytest; \
-	fi
-	[ -z "$(TEST_EXTRA_PACKAGES)" ] || pip install $(TEST_EXTRA_PACKAGES)
-
-install-style: install-docs install-dev
-	pip install pre-commit
+test:
+	$(PYTEST)
 
 style:
 	pre-commit run --all
@@ -91,8 +71,6 @@ install-docs:
 
 docs:
 	cd docs && make html
-
-linkcheck:
 	sphinx-build -W -b linkcheck ./docs/ ./docs/_build/linkcheck/
 
 release-deb:
@@ -105,12 +83,10 @@ release-deb:
 install-dev:
 	pip install -U pip setuptools wheel
 	pip install -e .
+	pip install -Ur test-requirements.txt $(TEST_EXTRA_PACKAGES)
+	pip install pre-commit
 	[ "$(ETESYNC_TESTS)" = "false" ] || pip install -Ue .[etesync]
-	set -xe && if [ "$(REQUIREMENTS)" = "devel" ]; then \
-	    pip install -U --force-reinstall \
-			git+https://github.com/mitsuhiko/click \
-			git+https://github.com/kennethreitz/requests; \
-	elif [ "$(REQUIREMENTS)" = "minimal" ]; then \
+	set -xe && if [ "$(REQUIREMENTS)" = "minimal" ]; then \
 		pip install -U --force-reinstall $$(python setup.py --quiet minimal_requirements); \
 	fi
 

docs/contributing.rst

@@ -87,8 +87,8 @@ virtualenv_ and run this inside of it::
     # Install git commit hook for some extra linting and checking
     pre-commit install
 
-    # install test dependencies
-    make install-test
+    # Install development dependencies
+    make install-dev
 
 Then you can run::
 
@@ -100,14 +100,13 @@ The ``Makefile`` has a lot of options that allow you to control which tests are
 run, and which servers are tested. Take a look at its code where they are all
 initialized and documented.
 
-For example, to test xandikos, first run the server itself::
-
-    docker run -p 8000:8000 whynothugo/vdirsyncer-devkit-xandikos
-
-Then run the tests specifying this ``DAV_SERVER``, run::
+To tests against a specific DAV server, use ``DAV_SERVER``::
 
     make DAV_SERVER=xandikos test
 
+The server will be initialised in a docker container and terminated at the end
+of the test suite.
+
 If you have any questions, feel free to open issues about it.
 
 Structure of the testsuite

docs/packaging.rst

@@ -41,9 +41,9 @@ repository or PyPI package. Trying to e.g. run ``pytest`` directly will
 require a lot of environment variables to be set (for configuration) and you
 probably don't want to deal with that.
 
-You can install the testing dependencies with::
+You can install the all development dependencies with::
 
-    make install-test
+    make install-dev
 
 You probably don't want this since it will use pip to download the
 dependencies. Alternatively you can find the testing dependencies in

docs/when.rst

@@ -50,7 +50,7 @@ program chosen:
 
 * Such a setup doesn't work at all with smartphones. Vdirsyncer, on the other
   hand, synchronizes with CardDAV/CalDAV servers, which can be accessed with
-  e.g. DAVDroid_ or the apps by dmfs_.
+  e.g. DAVx⁵_ or the apps by dmfs_.
 
-.. _DAVDroid: http://davdroid.bitfire.at/
+.. _DAVx⁵: https://www.davx5.com/
 .. _dmfs: https://dmfs.org/

setup.py

@@ -11,7 +11,7 @@
 
 requirements = [
     # https://github.com/mitsuhiko/click/issues/200
-    "click>=5.0",
+    "click>=5.0,<9.0",
     "click-log>=0.3.0, <0.4.0",
     # https://github.com/pimutils/vdirsyncer/issues/478
     "click-threading>=0.5",

tests/storage/conftest.py

@@ -1,6 +1,82 @@
+import contextlib
+import subprocess
+import time
 import uuid
 
 import pytest
+import requests
+
+
+def wait_for_container(url):
+    """Wait for a container to initialise.
+
+    Polls a URL every 100ms until the server responds.
+    """
+    # give the server 5 seconds to settle
+    for _ in range(50):
+        print(_)
+
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+        except requests.ConnectionError:
+            pass
+        else:
+            return
+
+        time.sleep(0.1)
+
+    pytest.exit(
+        "Server did not initialise in 5 seconds.\n"
+        "WARNING: There may be a stale docker container still running."
+    )
+
+
+@contextlib.contextmanager
+def dockerised_server(name, container_port, exposed_port):
+    """Run a dockerised DAV server as a contenxt manager."""
+    container_id = None
+    url = f"http://127.0.0.1:{exposed_port}/"
+
+    try:
+        # Hint: This will block while the pull happends, and only return once
+        # the container has actually started.
+        output = subprocess.check_output(
+            [
+                "docker",
+                "run",
+                "--detach",
+                "--publish",
+                f"{exposed_port}:{container_port}",
+                f"whynothugo/vdirsyncer-devkit-{name}",
+            ]
+        )
+
+        container_id = output.decode().strip()
+        wait_for_container(url)
+
+        yield url
+    finally:
+        if container_id:
+            subprocess.check_output(["docker", "kill", container_id])
+
+
+@pytest.fixture(scope="session")
+def baikal_server():
+    with dockerised_server("baikal", "80", "8002"):
+        yield
+
+
+@pytest.fixture(scope="session")
+def radicale_server():
+    with dockerised_server("radicale", "8001", "8001"):
+        yield
+
+
+@pytest.fixture(scope="session")
+def xandikos_server():
+    with dockerised_server("xandikos", "8000", "8000"):
+        yield
 
 
 @pytest.fixture

tests/storage/servers/baikal/__init__.py

@@ -3,7 +3,7 @@
 
 class ServerMixin:
     @pytest.fixture
-    def get_storage_args(self, request, tmpdir, slow_create_collection):
+    def get_storage_args(self, request, tmpdir, slow_create_collection, baikal_server):
         def inner(collection="test"):
             base_url = "http://127.0.0.1:8002/"
             args = {