Hotfix: Added debugpy support (#268)

Nusnus · web-flow · commit dda61c633394 · 2024-04-05T18:34:58.000+03:00
* Hotfix: Added debugpy support

* Docs
diff --git a/docs/devguide/local-development-environment.rst b/docs/devguide/local-development-environment.rst
@@ -45,6 +45,8 @@ To install the development dependencies, run the following command in your desir
         pip install -U pip ipython;\
         poetry install -E "all" --with test,dev,ci,docs
 
+.. _vscode:
+
 Debugging with VSCode
 =====================
 
@@ -64,7 +66,7 @@ Create a ``.vscode`` directory with the following ``launch.json`` file that can
         "configurations": [
             {
                 "name": "Python: Debug Tests",
-                "type": "python",
+                "type": "debugpy",
                 "request": "launch",
                 "program": "${file}",
                 "purpose": [
@@ -84,6 +86,21 @@ Create a ``.vscode`` directory with the following ``launch.json`` file that can
                 "stopOnEntry": false,
                 "showReturnValue": true,
                 "redirectOutput": true
+            },
+            {
+                "name": "Attach to Celery Worker",
+                "type": "debugpy",
+                "request": "attach",
+                "connect": {
+                    "host": "localhost",
+                    "port": 5678
+                },
+                "pathMappings": [
+                    {
+                        "localRoot": "${workspaceFolder}",
+                        "remoteRoot": "path-to-celery-source-in-container"
+                    }
+                ]
             }
         ]
     }
diff --git a/docs/getting-started/next-steps.rst b/docs/getting-started/next-steps.rst
@@ -258,11 +258,13 @@ And build our container using the standard `pytest-docker-tools <https://pypi.or
 
     myworker_container = container(
         image="{myworker_image.id}",
+        ports=MyWorkerContainer.ports(),
         environment=fxtr("default_worker_env"),
         network="{default_pytest_celery_network.name}",
         volumes={"{default_worker_volume.name}": defaults.DEFAULT_WORKER_VOLUME},
         wrapper_class=MyWorkerContainer,
         timeout=defaults.DEFAULT_WORKER_CONTAINER_TIMEOUT,
+        command=MyWorkerContainer.command(),
     )
 
 Pytest Integration
diff --git a/docs/userguide/celery-bug-report.rst b/docs/userguide/celery-bug-report.rst
@@ -361,6 +361,7 @@ We'll use the smoke tests worker to run the worker from the source code.
     from t.smoke.workers.dev import SmokeWorkerContainer
 
     from pytest_celery import RABBITMQ_PORTS
+    from pytest_celery import WORKER_DEBUGPY_PORTS
     from pytest_celery import CeleryBackendCluster
     from pytest_celery import CeleryBrokerCluster
     from pytest_celery import CeleryTestSetup
@@ -448,8 +449,14 @@ We'll use the smoke tests worker to run the worker from the source code.
                 "--without-gossip",
                 "--without-mingle",
                 "--without-heartbeat",
+                debugpy=False,
+                wait_for_client=True,
             )
 
+        @classmethod
+        def ports(cls) -> dict | None:
+            return WORKER_DEBUGPY_PORTS
+
 
     @pytest.fixture
     def default_worker_container_cls() -> type[SmokeWorkerContainer]:
diff --git a/docs/userguide/examples/django.rst b/docs/userguide/examples/django.rst
@@ -72,6 +72,8 @@ These will be used to configure the worker for the tests.
     ENV WORKER_NAME=$CELERY_WORKER_NAME
     ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+    EXPOSE 5678
+
 ``/src`` is arbitrarily chosen as the working directory to install the Django project from.
 
 .. code-block:: docker
@@ -133,6 +135,7 @@ and `container <https://github.com/Jc2k/pytest-docker-tools?tab=readme-ov-file#c
 
     default_worker_container = container(
         image="{worker_image.id}",
+        ports=fxtr("default_worker_ports"),
         environment=fxtr("default_worker_env"),
         network="{default_pytest_celery_network.name}",
         volumes={
diff --git a/docs/userguide/examples/myworker.rst b/docs/userguide/examples/myworker.rst
@@ -73,6 +73,8 @@ These will be used to configure the worker for the tests.
     ENV WORKER_NAME=$CELERY_WORKER_NAME
     ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+    EXPOSE 5678
+
 ``/src`` is arbitrarily chosen as the working directory to install Celery from.
 
 .. code-block:: docker
@@ -136,6 +138,22 @@ to the container instance.
         def worker_queue(cls) -> str:
             return "myworker"
 
+.. tip::
+
+    Add the following implementation to enable debugpy for the worker container.
+
+    .. code-block:: python
+
+        @classmethod
+        def ports(cls) -> dict | None:
+            return WORKER_DEBUGPY_PORTS
+
+        @classmethod
+        def command(cls, *args: str, **kwargs: dict) -> list[str]:
+            return super().command(*args, debugpy=True, wait_for_client=True, **kwargs)
+
+    The ``WORKER_DEBUGPY_PORTS`` can be imported from the plugin.
+
 Next, we build our worker image using the `build <https://github.com/Jc2k/pytest-docker-tools?tab=readme-ov-file#images>`_
 and `container <https://github.com/Jc2k/pytest-docker-tools?tab=readme-ov-file#containers>`_ fixtures.
 
@@ -156,11 +174,13 @@ These fixtures may be overridden if required.
 
     myworker_container = container(
         image="{myworker_image.id}",
+        ports=MyWorkerContainer.ports(),
         environment=fxtr("default_worker_env"),
         network="{default_pytest_celery_network.name}",
         volumes={"{default_worker_volume.name}": defaults.DEFAULT_WORKER_VOLUME},
         wrapper_class=MyWorkerContainer,
         timeout=defaults.DEFAULT_WORKER_CONTAINER_TIMEOUT,
+        command=MyWorkerContainer.command(),
     )
 
 Lastly, we wrap the container in a fixture to allow it to be injected into the test environment
diff --git a/docs/userguide/examples/range.rst b/docs/userguide/examples/range.rst
@@ -144,6 +144,7 @@ Our focus should be on the ``worker_containers`` list, which will contain the na
             cnt = f"worker_v{v.replace('.', '_')}_container"
             globals()[cnt] = container(
                 image="{" + f"{img}.id" + "}",
+                ports=fxtr("default_worker_ports"),
                 environment=fxtr("default_worker_env"),
                 network="{default_pytest_celery_network.name}",
                 volumes={"{default_worker_volume.name}": DEFAULT_WORKER_VOLUME},
diff --git a/examples/django/tests/DjangoWorker.Dockerfile b/examples/django/tests/DjangoWorker.Dockerfile
@@ -14,6 +14,8 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL
 ENV WORKER_NAME=$CELERY_WORKER_NAME
 ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+EXPOSE 5678
+
 # Install packages
 WORKDIR /src
 
diff --git a/examples/django/tests/conftest.py b/examples/django/tests/conftest.py
@@ -45,6 +45,7 @@ def worker_queue(cls) -> str:
 
 default_worker_container = container(
     image="{worker_image.id}",
+    ports=fxtr("default_worker_ports"),
     environment=fxtr("default_worker_env"),
     network="{default_pytest_celery_network.name}",
     volumes={
diff --git a/examples/myworker/tests/myworker/Dockerfile b/examples/myworker/tests/myworker/Dockerfile
@@ -14,6 +14,8 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL
 ENV WORKER_NAME=$CELERY_WORKER_NAME
 ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+EXPOSE 5678
+
 # Install packages
 WORKDIR /src
 
diff --git a/examples/myworker/tests/myworker/myworker.py b/examples/myworker/tests/myworker/myworker.py
@@ -45,11 +45,13 @@ def worker_queue(cls) -> str:
 
 myworker_container = container(
     image="{myworker_image.id}",
+    ports=MyWorkerContainer.ports(),
     environment=fxtr("default_worker_env"),
     network="{default_pytest_celery_network.name}",
     volumes={"{default_worker_volume.name}": defaults.DEFAULT_WORKER_VOLUME},
     wrapper_class=MyWorkerContainer,
     timeout=defaults.DEFAULT_WORKER_CONTAINER_TIMEOUT,
+    command=MyWorkerContainer.command(),
 )
 
 
diff --git a/examples/range/tests/test_range_cluster.py b/examples/range/tests/test_range_cluster.py
@@ -34,6 +34,7 @@ def generate_workers(versions: list[str]) -> list[str]:
         cnt = f"worker_v{v.replace('.', '_')}_container"
         globals()[cnt] = container(
             image="{" + f"{img}.id" + "}",
+            ports=fxtr("default_worker_ports"),
             environment=fxtr("default_worker_env"),
             network="{default_pytest_celery_network.name}",
             volumes={"{default_worker_volume.name}": DEFAULT_WORKER_VOLUME},
diff --git a/examples/worker_pool/Dockerfile b/examples/worker_pool/Dockerfile
@@ -14,6 +14,8 @@ ENV LOG_LEVEL=$CELERY_LOG_LEVEL
 ENV WORKER_NAME=$CELERY_WORKER_NAME
 ENV WORKER_QUEUE=$CELERY_WORKER_QUEUE
 
+EXPOSE 5678
+
 # Install packages
 COPY --chown=test_user:test_user requirements.txt .
 RUN pip install --no-cache-dir --upgrade pip
diff --git a/examples/worker_pool/tests/test_gevent_pool.py b/examples/worker_pool/tests/test_gevent_pool.py
@@ -35,6 +35,7 @@ def command(cls, *args: str) -> list[str]:
 
 gevent_worker_container = container(
     image="{gevent_worker_image.id}",
+    ports=fxtr("default_worker_ports"),
     environment=fxtr("default_worker_env"),
     network="{default_pytest_celery_network.name}",
     volumes={"{default_worker_volume.name}": defaults.DEFAULT_WORKER_VOLUME},
diff --git a/src/pytest_celery/api/container.py b/src/pytest_celery/api/container.py
@@ -54,14 +54,23 @@ def celeryconfig(self) -> dict:
         raise NotImplementedError("CeleryTestContainer.celeryconfig")
 
     @classmethod
-    def command(cls, *args: str) -> list[str]:
+    def command(
+        cls,
+        *args: str,
+        debugpy: bool = False,
+        wait_for_client: bool = True,
+        **kwargs: dict,
+    ) -> list[str]:
         """Override the CMD instruction in the Dockerfile.
 
         This method should be overridden in derived classes to provide the
         specific command and its arguments required to start the container.
 
         Args:
             *args (str): Additional command-line arguments.
+            debugpy (bool): Enable debugpy. Defaults to False.
+            wait_for_client (bool): Wait for a debugger to be attached. Defaults to True.
+            **kwargs (dict): Additional keyword arguments.
 
         Raises:
             NotImplementedError: Rely on the Dockerfile if not set otherwise by default.
diff --git a/src/pytest_celery/vendors/redis/container.py b/src/pytest_celery/vendors/redis/container.py
@@ -37,7 +37,13 @@ def celeryconfig(self) -> dict:
         }
 
     @classmethod
-    def command(cls, *args: str) -> list[str]:
+    def command(
+        cls,
+        *args: str,
+        debugpy: bool = False,
+        wait_for_client: bool = True,
+        **kwargs: dict,
+    ) -> list[str]:
         return ["redis-server", *args]
 
     @property
diff --git a/src/pytest_celery/vendors/worker/container.py b/src/pytest_celery/vendors/worker/container.py
@@ -34,20 +34,48 @@ class CeleryWorkerContainer(CeleryTestContainer):
     """
 
     @classmethod
-    def command(cls, *args: str) -> list[str]:
+    def command(
+        cls,
+        *args: str,
+        debugpy: bool = False,
+        wait_for_client: bool = True,
+        **kwargs: dict,
+    ) -> list[str]:
         args = args or tuple()
-        return [
-            "celery",
-            "-A",
-            "app",
-            "worker",
-            f"--loglevel={cls.log_level()}",
-            "-n",
-            f"{cls.worker_name()}@%h",
-            "-Q",
-            f"{cls.worker_queue()}",
-            *args,
-        ]
+        cmd = list()
+
+        if debugpy:
+            cmd.extend(
+                [
+                    "python",
+                    "-m",
+                    "debugpy",
+                    "--listen",
+                    "0.0.0.0:5678",
+                ]
+            )
+
+            if wait_for_client:
+                cmd.append("--wait-for-client")
+
+            cmd.append("-m")
+
+        cmd.extend(
+            [
+                "celery",
+                "-A",
+                "app",
+                "worker",
+                f"--loglevel={cls.log_level()}",
+                "-n",
+                f"{cls.worker_name()}@%h",
+                "-Q",
+                f"{cls.worker_queue()}",
+                *args,
+            ]
+        )
+
+        return cmd
 
     def _wait_port(self, port: str) -> int:
         # Not needed for worker container
