diff --git a/MANIFEST.in b/MANIFEST.in
index f794f78f..a6286815 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,2 +1,4 @@
 include versioneer.py
 include jupyterhub_traefik_proxy/_version.py
+include requirements.txt
+recursive-include tests *.*
diff --git a/setup.py b/setup.py
index 08434d1a..ec622e22 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
     ],
-    packages=find_packages(),
+    packages=find_packages(exclude=["performance", "tests"]),
     include_package_data=True,
     entry_points={
         "jupyterhub.proxies": [
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/conftest.py b/tests/conftest.py
index 9d62cb0f..e35c0daf 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,7 +13,6 @@
 from urllib.parse import urlparse
 
 import pytest
-import utils
 from certipy import Certipy
 from consul.aio import Consul
 from jupyterhub.utils import exponential_backoff
@@ -24,6 +23,8 @@
 from jupyterhub_traefik_proxy.fileprovider import TraefikFileProviderProxy
 from jupyterhub_traefik_proxy.traefik_utils import deep_merge
 
+from . import utils
+
 HERE = Path(__file__).parent.resolve()
 config_files = HERE / "config_files"
 
diff --git a/tests/test_proxy.py b/tests/test_proxy.py
index 3cb6ea46..d7099679 100644
--- a/tests/test_proxy.py
+++ b/tests/test_proxy.py
@@ -14,7 +14,6 @@
 from urllib.parse import quote, urlparse
 
 import pytest
-import utils
 import websockets
 from jupyterhub.objects import Hub, Server
 from jupyterhub.user import User
@@ -23,6 +22,8 @@
 
 from jupyterhub_traefik_proxy.proxy import TraefikProxy
 
+from . import utils
+
 # Mark all tests in this file as slow
 pytestmark = [pytest.mark.slow]