feat: Add edxapp celery worker services (optional) (#69)

Copied worker configs from Confluence with the following changes:

- Removed newrelic integration
- Added mysql80 dependency for workes (not just mysql57)
- Updated image to match the newer coordinates (matching lms and cms
  containers, which apparently both use `lms-dev`)
- Added explanation to `C_FORCE_ROOT`
- Fixed cms-worker hostname (was using `lms` in name)

Supporting changes:

- Add docs and comment explaining how to use the workers
- Add cms-worker and lms-worker to the list of services available for
  CLI autocomplete

Some mostly unrelated cleanup as well:

- Update lms and cms Django settings argument from `devstack_docker`
  to just `devstack`; the former only imported the latter and that has
  been the case for years (after the switch from Vagrant).
- Alphabetize service lists (cms was renamed from studio and should have
  been moved)
- Fix some link syntax in the service list

Author: timmc-edx

docker-compose-host.yml

@@ -29,6 +29,9 @@ services:
       - edxapp_tox:/edx/app/edxapp/edx-platform/.tox
       - edxapp_uploads:/edx/var/edxapp/uploads
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
+  lms-worker:
+    volumes:
+      - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
   edx_notes_api:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-notes-api:/edx/app/notes/
@@ -47,6 +50,9 @@ services:
       - edxapp_tox:/edx/app/edxapp/edx-platform/.tox
       - edxapp_uploads:/edx/var/edxapp/uploads
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
+  cms-worker:
+    volumes:
+      - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
   insights:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-analytics-dashboard:/edx/app/insights/insights

docker-compose.yml

@@ -406,7 +406,8 @@ services:
       - "44567:4567"
 
   lms:
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack_docker; sleep 2; done'
+    # Switch to `--settings devstack_with_worker` if you want to use lms-worker
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms"
     hostname: lms.devstack.edx
     depends_on:
@@ -446,6 +447,22 @@ services:
     volumes:
       - edxapp_lms_assets:/edx/var/edxapp/staticfiles/
 
+  lms-worker:
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && celery --app=lms.celery:APP worker -l debug -c 2'
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms-worker"
+    hostname: lms-worker.devstack.edx
+    depends_on:
+      - mysql80
+      - mysql57
+      - redis
+    stdin_open: true
+    tty: true
+    image: edxops/lms-dev:latest
+    environment:
+      DJANGO_SETTINGS_MODULE: lms.envs.devstack_with_worker
+      # Dangerous to run Celery as root normally, but it's how we do things in devstack for some reason
+      C_FORCE_ROOT: "true"
+
   insights:
     command: bash -c 'source /edx/app/insights/insights_env && while true; do python /edx/app/insights/insights/manage.py runserver 0.0.0.0:18110 --settings analytics_dashboard.settings.devstack; sleep 2; done'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.insights"
@@ -584,7 +601,8 @@ services:
       - /edx/var/registrar/
 
   cms:
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack_docker; sleep 2; done'
+    # Switch to `--settings devstack_with_worker` if you want to use cms-worker
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack; sleep 2; done'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms"
     hostname: cms.devstack.edx
     depends_on:
@@ -623,6 +641,22 @@ services:
     volumes:
       - edxapp_cms_assets:/edx/var/edxapp/staticfiles/
 
+  cms-worker:
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform && celery --app=cms.celery:APP worker -l debug -c 2'
+    container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms-worker"
+    hostname: cms-worker.devstack.edx
+    depends_on:
+      - mysql80
+      - mysql57
+      - redis
+    stdin_open: true
+    tty: true
+    image: edxops/lms-dev:latest
+    environment:
+      DJANGO_SETTINGS_MODULE: cms.envs.devstack_with_worker
+      # Dangerous to run Celery as root normally, but it's how we do things in devstack for some reason
+      C_FORCE_ROOT: "true"
+
   xqueue:
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.xqueue"
     image: edxops/xqueue-dev:latest

docs/advanced_configuration.rst

@@ -25,3 +25,13 @@ This is handled for you automatically by setting the ``OPENEDX_RELEASE`` environ
 
 As a specific example, if ``OPENEDX_RELEASE`` is set in your environment as ``juniper.master``, then ``COMPOSE_PROJECT_NAME`` will default to ``devstack-juniper.master`` instead of ``devstack``.
 
+Celery workers
+~~~~~~~~~~~~~~
+
+In devstack, edxapp and possibly other services are configured to run Celery tasks synchronously rather than requiring another worker service to handle them. However, there is some support for using a separate worker. To use Celery for LMS or CMS:
+
+1. Edit ``docker-compose.yml`` to update the LMS and CMS ``command`` lines to use ``--settings devstack_with_worker`` instead of just ``--settings devstack``
+2. Run ``make lms-worker-up cms-worker-up``
+3. Start LMS and CMS as usual
+
+To verify that Celery communication is working, try ``curl -sS 'http://localhost:18000/heartbeat?extended'`` and confirm that the ``celery`` component of the response shows success.

docs/service_list.rst

@@ -55,13 +55,13 @@ Instead of a service name or list, you can also run commands like ``make dev.pro
 +------------------------------------+-------------------------------------+----------------+--------------+
 | `xqueue`_                          | http://localhost:18040/api/v1/      | Python/Django  | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
-| `coursegraph`                      | http://localhost:7474/browser       | Tooling (Java) | Extra        |
+| `coursegraph`_                     | http://localhost:7474/browser       | Tooling (Java) | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
-| `insights`                         | http://localhost:18110              | Python/Django  | Extra        |
+| `insights`_                        | http://localhost:18110              | Python/Django  | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
-| `analyticsapi`                     | http://localhost:19001              | Python/Django  | Extra        |
+| `analyticsapi`_                    | http://localhost:19001              | Python/Django  | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
-| `frontend-app-ora-grading`         | http://localhost:1993               | MFE (React.js) | Extra        |
+| `frontend-app-ora-grading`_        | http://localhost:1993               | MFE (React.js) | Extra        |
 +------------------------------------+-------------------------------------+----------------+--------------+
 
 Some common service combinations include:

options.mk

@@ -67,7 +67,7 @@ credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-authn+frontend-
 # Separated by plus signs.
 # Separated by plus signs. Listed in alphabetical order for clarity.
 EDX_SERVICES ?= \
-analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms_watcher+registrar+registrar-worker+cms+cms_watcher+xqueue+xqueue_consumer
+analyticsapi+credentials+cms+cms-worker+cms_watcher+discovery+ecommerce+edx_notes_api+forum+frontend-app-account+frontend-app-learner-dashboard+frontend-app-learner-record+frontend-app-profile+frontend-app-authn+frontend-app-course-authoring+frontend-app-gradebook+frontend-app-ora-grading+frontend-app-learning+frontend-app-library-authoring+frontend-app-payment+frontend-app-program-console+frontend-app-publisher+insights+lms+lms-worker+lms_watcher+registrar+registrar-worker+xqueue+xqueue_consumer
 
 # Services with database migrations.
 # Should be a subset of $(EDX_SERVICES).
@@ -76,7 +76,7 @@ analyticsapi+credentials+discovery+ecommerce+edx_notes_api+forum+frontend-app-ac
 # Note: This list should contain _all_ db-backed services, even if not
 # configured to run; the list will be filtered later against $(DEFAULT_SERVICES).
 DB_SERVICES ?= \
-credentials+discovery+ecommerce+lms+registrar+cms
+credentials+cms+discovery+ecommerce+lms+registrar
 
 # Services with static assets to be built.
 # Should be a subset of $(EDX_SERVICES).
@@ -85,7 +85,7 @@ credentials+discovery+ecommerce+lms+registrar+cms
 # Note: This list should contain _all_ services with static asse to compile ts, even if not
 # configured to run; the list will be filtered later against $(DEFAULT_SERVICES).
 ASSET_SERVICES ?= \
-credentials+discovery+ecommerce+insights+lms+registrar+cms
+credentials+cms+discovery+ecommerce+insights+lms+registrar
 
 # All third-party services.
 # Separated by plus signs. Listed in alphabetical order for clarity.