fix: For the openedx-events broker, use kafka _everywhere_ instead of redis

Devstack is currently split between services pre-configured to use
redis, and others pre-configured to use kafka.  This results in the
exceedingly dumb result of events produced via one broker, but never
consumed because the consuming apps are hooked up to the other broker.

Author: pwnage101

docker-compose-host.yml

@@ -41,6 +41,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
       - ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py
       - ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py
+      - ${PWD}/py_private_requirements/lms.txt:/edx/private_requirements.txt
   lms-worker:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
@@ -71,6 +72,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
       - ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py
       - ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py
+      - ${PWD}/py_private_requirements/cms.txt:/edx/private_requirements.txt
   cms-worker:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
@@ -95,7 +97,11 @@ services:
       - ${DEVSTACK_WORKSPACE}/enterprise-subsidy:/edx/app/enterprise-subsidy
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
       - ${PWD}/py_configuration_files/enterprise_subsidy.py:/edx/app/enterprise-subsidy/enterprise_subsidy/settings/devstack.py
-
+  enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle:
+    volumes:
+      - ${DEVSTACK_WORKSPACE}/enterprise-subsidy:/edx/app/enterprise-subsidy
+      - ${DEVSTACK_WORKSPACE}/src:/edx/src
+      - ${PWD}/py_configuration_files/enterprise_subsidy.py:/edx/app/enterprise-subsidy/enterprise_subsidy/settings/devstack.py
   enterprise-catalog:
     volumes:
       - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise_catalog/enterprise_catalog

docker-compose.yml

@@ -449,7 +449,7 @@ services:
 
   lms:
     # Switch to `--settings devstack_with_worker` if you want to use lms-worker
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done'
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms"
     hostname: lms.devstack.edx
     depends_on:
@@ -638,7 +638,7 @@ services:
 
   cms:
     # Switch to `--settings devstack_with_worker` if you want to use cms-worker
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack; sleep 2; done'
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms"
     hostname: cms.devstack.edx
     depends_on:
@@ -905,6 +905,22 @@ services:
     volumes:
       - /edx/var/enterprise-subsidy
 
+  enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle:
+    image: edxops/enterprise-subsidy-dev
+    container_name: edx.devstack.enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle
+    depends_on:
+      - mysql80
+      - memcached
+    command: >-
+      bash -c 'while true; do python /edx/app/enterprise-subsidy/manage.py consume_events -t learner-credit-course-enrollment-lifecycle -g enterprise_subsidy_dev; sleep 2; done'
+    tty: true
+    environment:
+      DJANGO_SETTINGS_MODULE: enterprise_subsidy.settings.devstack
+    working_dir: /edx/app/enterprise-subsidy
+    ports:
+      - "18281:18281"
+    stdin_open: true
+
   # ==========================================================================
   # edX Microfrontends
   #

py_configuration_files/cms.py

@@ -297,10 +297,11 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 
 #################### Event bus backend ########################
 
-EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
-EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
-EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
 
 course_catalog_event_setting = EVENT_BUS_PRODUCER_CONFIG['org.openedx.content_authoring.course.catalog_info.changed.v1']
 course_catalog_event_setting['course-catalog-info-changed']['enabled'] = True

py_configuration_files/course_discovery.py

@@ -86,9 +86,10 @@
 
 CELERY_TASK_ALWAYS_EAGER = False
 
-EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
-EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
-EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
 
 #####################################################################

py_configuration_files/lms.py

@@ -80,7 +80,10 @@
 
 ################################ DEBUG TOOLBAR ################################
 
-INSTALLED_APPS += ['debug_toolbar']
+INSTALLED_APPS += (
+    'debug_toolbar',
+    'edx_event_bus_kafka',
+)
 MIDDLEWARE += [
     'lms.djangoapps.discussion.django_comment_client.utils.QueryCountDebugMiddleware',
     'debug_toolbar.middleware.DebugToolbarMiddleware',
@@ -504,10 +507,11 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 CLOSEST_CLIENT_IP_FROM_HEADERS = []
 
 #################### Event bus backend ########################
-EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
-EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
-EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
 
 certificate_revoked_event_config = EVENT_BUS_PRODUCER_CONFIG['org.openedx.learning.certificate.revoked.v1']
 certificate_revoked_event_config['learning-certificate-lifecycle']['enabled'] = True

py_private_requirements/cms.txt

@@ -0,0 +1 @@
+confluent-kafka[avro,schema-registry]==2.10.0

py_private_requirements/lms.txt

@@ -0,0 +1 @@
+confluent-kafka[avro,schema-registry]==2.10.0