Merge pull request #142 from edx/pwnage101/use-kafka-everywhere

fix: For the openedx-events broker, use kafka _everywhere_ instead of redis

Author: pwnage101

docker-compose-host.yml

@@ -43,6 +43,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
       - ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py
       - ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py
+      - ${PWD}/py_private_requirements/lms.txt:/edx/private_requirements.txt
   lms-worker:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
@@ -73,6 +74,7 @@ services:
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
       - ${PWD}/py_configuration_files/cms.py:/edx/app/edxapp/edx-platform/cms/envs/devstack.py
       - ${PWD}/py_configuration_files/lms.py:/edx/app/edxapp/edx-platform/lms/envs/devstack.py
+      - ${PWD}/py_private_requirements/cms.txt:/edx/private_requirements.txt
   cms-worker:
     volumes:
       - ${DEVSTACK_WORKSPACE}/edx-platform:/edx/app/edxapp/edx-platform
@@ -97,7 +99,11 @@ services:
       - ${DEVSTACK_WORKSPACE}/enterprise-subsidy:/edx/app/enterprise-subsidy
       - ${DEVSTACK_WORKSPACE}/src:/edx/src
       - ${PWD}/py_configuration_files/enterprise_subsidy.py:/edx/app/enterprise-subsidy/enterprise_subsidy/settings/devstack.py
-
+  enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle:
+    volumes:
+      - ${DEVSTACK_WORKSPACE}/enterprise-subsidy:/edx/app/enterprise-subsidy
+      - ${DEVSTACK_WORKSPACE}/src:/edx/src
+      - ${PWD}/py_configuration_files/enterprise_subsidy.py:/edx/app/enterprise-subsidy/enterprise_subsidy/settings/devstack.py
   enterprise-catalog:
     volumes:
       - ${DEVSTACK_WORKSPACE}/enterprise-catalog:/edx/app/enterprise-catalog

docker-compose.yml

@@ -449,7 +449,7 @@ services:
 
   lms:
     # Switch to `--settings devstack_with_worker` if you want to use lms-worker
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done'
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.lms"
     hostname: lms.devstack.edx
     depends_on:
@@ -638,7 +638,7 @@ services:
 
   cms:
     # Switch to `--settings devstack_with_worker` if you want to use cms-worker
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && while true; do python /edx/app/edxapp/edx-platform/manage.py cms runserver 0.0.0.0:18010 --settings devstack; sleep 2; done'
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && (pip install -r /edx/private_requirements.txt; while true; do python /edx/app/edxapp/edx-platform/manage.py lms runserver 0.0.0.0:18000 --settings devstack; sleep 2; done)'
     container_name: "edx.${COMPOSE_PROJECT_NAME:-devstack}.cms"
     hostname: cms.devstack.edx
     depends_on:
@@ -906,6 +906,22 @@ services:
     volumes:
       - /edx/var/enterprise-subsidy
 
+  enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle:
+    image: edxops/enterprise-subsidy-dev
+    container_name: edx.devstack.enterprise-subsidy-consume_learner_credit_course_enrollment_lifecycle
+    depends_on:
+      - mysql80
+      - memcached
+    command: >-
+      bash -c 'while true; do python /edx/app/enterprise-subsidy/manage.py consume_events -t learner-credit-course-enrollment-lifecycle -g enterprise_subsidy_dev; sleep 2; done'
+    tty: true
+    environment:
+      DJANGO_SETTINGS_MODULE: enterprise_subsidy.settings.devstack
+    working_dir: /edx/app/enterprise-subsidy
+    ports:
+      - "18281:18281"
+    stdin_open: true
+
   # ==========================================================================
   # edX Microfrontends
   #

py_configuration_files/cms.py

@@ -90,7 +90,7 @@
 
 ################################ DEBUG TOOLBAR ################################
 
-INSTALLED_APPS += ['debug_toolbar']
+INSTALLED_APPS += ('debug_toolbar',)
 
 MIDDLEWARE.append('debug_toolbar.middleware.DebugToolbarMiddleware')
 INTERNAL_IPS = ('127.0.0.1',)
@@ -296,11 +296,12 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 ]
 
 #################### Event bus backend ########################
-
-EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
-EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
+INSTALLED_APPS += ('edx_event_bus_kafka',)
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
-EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
 
 course_catalog_event_setting = EVENT_BUS_PRODUCER_CONFIG['org.openedx.content_authoring.course.catalog_info.changed.v1']
 course_catalog_event_setting['course-catalog-info-changed']['enabled'] = True

py_configuration_files/course_discovery.py

@@ -86,9 +86,12 @@
 
 CELERY_TASK_ALWAYS_EAGER = False
 
-EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
-EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
-EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
+#################### Event bus backend ########################
+INSTALLED_APPS += ('edx_event_bus_kafka',)
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
 
 #####################################################################

py_configuration_files/enterprise_access.py

@@ -61,10 +61,7 @@
 
 # Install django-extensions for improved dev experiences
 # https://github.com/django-extensions/django-extensions#using-it
-INSTALLED_APPS += (
-    'django_extensions',
-    'edx_event_bus_kafka',
-)
+INSTALLED_APPS += ('django_extensions',)
 
 # BEGIN CELERY
 CELERY_WORKER_HIJACK_ROOT_LOGGER = True
@@ -114,6 +111,7 @@
 ################### Kafka Related Settings ##############################
 
 # "Standard" Kafka settings as defined in https://github.com/openedx/event-bus-kafka/tree/main
+INSTALLED_APPS += ('edx_event_bus_kafka',)
 EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
 EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
 EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'

py_configuration_files/enterprise_subsidy.py

@@ -33,10 +33,6 @@
 
 }
 
-INSTALLED_APPS += (
-    'edx_event_bus_kafka',
-)
-
 # Generic OAuth2 variables irrespective of SSO/backend service key types.
 OAUTH2_PROVIDER_URL = 'http://edx.devstack.lms:18000/oauth2'
 
@@ -83,6 +79,7 @@
 
 # Kafka Settings
 # "Standard" Kafka settings as defined in https://github.com/openedx/event-bus-kafka/tree/main
+INSTALLED_APPS += ('edx_event_bus_kafka',)
 EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
 EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
 EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'

py_configuration_files/lms.py

@@ -80,7 +80,7 @@
 
 ################################ DEBUG TOOLBAR ################################
 
-INSTALLED_APPS += ['debug_toolbar']
+INSTALLED_APPS += ('debug_toolbar',)
 MIDDLEWARE += [
     'lms.djangoapps.discussion.django_comment_client.utils.QueryCountDebugMiddleware',
     'debug_toolbar.middleware.DebugToolbarMiddleware',
@@ -504,10 +504,12 @@ def should_show_debug_toolbar(request):  # lint-amnesty, pylint: disable=missing
 CLOSEST_CLIENT_IP_FROM_HEADERS = []
 
 #################### Event bus backend ########################
-EVENT_BUS_PRODUCER = 'edx_event_bus_redis.create_producer'
-EVENT_BUS_REDIS_CONNECTION_URL = 'redis://:[email protected]:6379/'
+INSTALLED_APPS += ('edx_event_bus_kafka',)
+EVENT_BUS_KAFKA_SCHEMA_REGISTRY_URL = 'http://edx.devstack.schema-registry:8081'
+EVENT_BUS_KAFKA_BOOTSTRAP_SERVERS = 'edx.devstack.kafka:29092'
+EVENT_BUS_PRODUCER = 'edx_event_bus_kafka.create_producer'
+EVENT_BUS_CONSUMER = 'edx_event_bus_kafka.KafkaEventConsumer'
 EVENT_BUS_TOPIC_PREFIX = 'dev'
-EVENT_BUS_CONSUMER = 'edx_event_bus_redis.RedisEventConsumer'
 
 certificate_revoked_event_config = EVENT_BUS_PRODUCER_CONFIG['org.openedx.learning.certificate.revoked.v1']
 certificate_revoked_event_config['learning-certificate-lifecycle']['enabled'] = True

py_private_requirements/cms.txt

@@ -0,0 +1 @@
+confluent-kafka[avro,schema-registry]==2.10.0

py_private_requirements/lms.txt

@@ -0,0 +1 @@
+confluent-kafka[avro,schema-registry]==2.10.0