Enable remote debugging using VSCode - see HEA-760

This changes allows setting LAUNCHER in .env
and then using VSCode to attach to the Python
process running inside the Docker container(s)

Author: rhunwicks

README.md

@@ -23,3 +23,53 @@ baseline
 
 This produces a .puml file that can be rendered using a PlantUML
 server, either within your IDE or using a service like http://www.plantuml.com/.
+
+## Debugging inside Docker Containers
+
+The `LAUNCHER` environment sets a wrapper program around the Python process
+(`gunicorn`, `dagster-daemon`, `dagster-webserver`). This can be used to
+enable a debugger inside Docker Containers:
+
+1. Set `LAUNCHER="python3 -m debugpy --listen 0.0.0.0:5678"` in `.env`
+2. Create Launch Configurations in Visual Studio Code like:
+
+```json
+			{
+				"name": "Python: Attach to app (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5678
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			},
+			{
+				"name": "Python: Attach to dagster-daemon (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5680
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			}
+```
+
+3. Start the Docker containers as normal, and then use the Run and Debug
+pane in Visual Studio code to launch the configuration that attaches to
+the desired server.

docker-compose.override.yml

@@ -15,6 +15,7 @@ services:
     build:
       target: dev
     ports:
+      - "5678:5678"
       - "8000:8000"
       - "8888:8888"
     volumes:
@@ -30,6 +31,9 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
       # Put .coverage in a writable directory
       COVERAGE_FILE: log/.coverage
     command:
@@ -41,6 +45,7 @@ services:
     restart: no
     ports:
       - "3000:3000"
+      - "5679:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -54,9 +59,14 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 
   dagster_daemon:
     restart: no
+    ports:
+      - "5680:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -70,4 +80,7 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 

docker/app/run_dagster_daemon.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-daemon run $*
+echo Starting Dagster Daemon
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-daemon run $*

docker/app/run_dagster_webserver.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*
+echo Starting Dagster Webserver
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*

docker/app/run_django.sh

@@ -40,7 +40,10 @@ touch log/django_sql.log
 chown -R django:django log/*
 
 echo Starting Gunicorn with DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE}
-gosu django  gunicorn ${APP}.wsgi:application \
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/gunicorn ${APP}.wsgi:application \
     --name ${APP}${ENV} \
     --config $(dirname $(readlink -f "$0"))/gunicorn_config.py \
     $* 2>&1

env.example

@@ -48,3 +48,8 @@ BSS_METADATA_WORKBOOK='gdrive://Database Design/BSS Metadata'  # 15XVXFjbom1sScV
 BSS_METADATA_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
 BSS_FILES_FOLDER='gdrive://Discovery Folder/Baseline Storage Sheets (BSS)'
 BSS_FILES_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
+
+# LAUNCHER can be used to configure a wrapper program around the Python process
+# For example, to add ddtrace or debugpy
+# Use the VSCode debugger as a launcher
+# LAUNCHER = "python3 -m debugpy --listen 0.0.0.0:5679"

requirements/local.txt

@@ -1,2 +1,3 @@
 -r test.txt
 -r lint.txt
+debugpy