Fix #3 support elasticsearch-dsl-py v6.4 and fix bug in es-list - PD-25495

Makefile

@@ -74,3 +74,31 @@ selfcheck: ## check that the Makefile is well-formed
 
 pylintrc: ## check that the Makefile is well-formed
 	edx_lint write pylintrc
+
+nuke:
+	@echo "Removing containers, networks, and data... 🧨"
+	@echo
+	docker compose down -v
+
+es-health-local-api:
+	@echo "Assuming you've got docker up, getting health of elasticsearch... 📄"
+	@echo
+	curl 'localhost:9200/_cat/health?format=json&pretty=true'
+
+es-list-indexes-local-api:
+	@echo "Assuming you've got docker up, getting elasticsearch indexes via API call... 📄"
+	curl 'localhost:9200/_cat/indices?format=json&pretty=true'
+
+es-print-java-env:
+	@echo "printing runtime value of ES_JAVA_OPTS"
+	docker-compose run --rm elastic printenv ES_JAVA_OPTS
+
+up:
+	@echo "Bringing up elastic 6 in docker 🚀"
+	docker compose up -d --remove-orphans
+
+down:
+	@echo "Destroying local containers, but keeping data volumes intact... 😴"
+	docker compose down
+
+

django_elastic_migrations/indexes.py

@@ -7,7 +7,7 @@
 from django.db import ProgrammingError
 from elasticsearch import TransportError
 from elasticsearch.helpers import expand_action, bulk
-from elasticsearch_dsl import Index as ESIndex, DocType as ESDocType, Q as ESQ, Search
+from elasticsearch_dsl import Index as ESIndex, Document as ESDocument, Q as ESQ, Search
 
 from django_elastic_migrations import es_client, environment_prefix, es_test_prefix, dem_index_paths, get_logger, codebase_id
 from django_elastic_migrations.exceptions import DEMIndexNotFound, DEMDocTypeRequiresGetReindexIterator, \
@@ -478,7 +478,7 @@ def __getattribute__(self, item):
         return None
 
 
-class DEMDocType(ESDocType):
+class DEMDocType(ESDocument):
     """
     Django users subclass DEMDocType instead of Elasticsearch's DocType
     to use Django Elastic Migrations. All documentation from their class
@@ -771,6 +771,24 @@ def batched_bulk_index(cls, queryset=None, workers=0, last_updated_datetime=None
                     num_failures += result_failures
         return num_successes, num_failures
 
+    def _get_index(self, index=None, required=True):
+        """
+        We have chosen to override this method to allow for the use of
+        a dynamic index at runtime. This is necessary to support the
+        key use case of Django Elastic Migrations: being able to index
+        into a new index version while the old index version is still
+        being used by the application.
+
+        A future version of DEM may instead elect to use a different
+        technique to supply this information. The inheritance hierarchy
+        for elasticsearch-dsl-py is not very friendly to overriding.
+        """
+        if index is None:
+            index = self.get_dem_index().get_es_index_name()
+        if index is None:
+            index = super(DEMDocType, self)._get_index(index, required)
+        return index
+
 
 class DEMIndex(ESIndex):
     """

django_elastic_migrations/management/commands/es.py

@@ -122,7 +122,7 @@ def get_index_specifying_options(cls, options, require_one_include_list=None):
         if apply_all and indexes:
             logger.warning(
                 "Received --all along with index names: '{indexes}'."
-                "Noramlly you would not specify names of indexes "
+                "Normally you would not specify names of indexes "
                 "with --all, since --all covers all the indexes. "
                 "The --all has been canceled; operating on just '{indexes}'."
                 "To clear *all* the indexes, just use --all.".format(

django_elastic_migrations/management/commands/es_list.py

@@ -64,7 +64,6 @@ def handle(self, *args, **options):
                 for dem_index in indexes:
                     dem_index_model = dem_index.get_index_model()
                     index_versions = dem_index_model.get_available_versions_with_prefix()
-                    row = None
                     if index_versions:
                         for index_version in index_versions:
                             num_docs = DEMIndexManager.get_es_index_doc_count(index_version.name)
@@ -74,9 +73,9 @@ def handle(self, *args, **options):
                                             index_version.is_active or 0,
                                             num_docs,
                                             index_version.tag)
+                            rows.append(row)
                     else:
                         row = EsListRow(dem_index.get_base_name(), "", False, False, 0, "Current (not created)")
-                    if row:
                         rows.append(row)
             except AttributeError:
                 raise FirstMigrationNotRunError()

django_elastic_migrations/models.py

@@ -30,7 +30,7 @@ class Index(models.Model):
     """
     name = models.CharField(verbose_name="Index Name", max_length=32, unique=True)
 
-    # Django convention is to use '+' for related name when you don't need the
+    # Django's convention is to use '+' for related name when you don't need the
     # reverse relation. in this case, we already have IndexVersion pointing
     # back to Index, so we don't need that reverse name.
     # See https://docs.djangoproject.com/en/2.0/ref/models/fields/#django.db.models.ForeignKey.related_name

docker-compose.yml

@@ -10,7 +10,8 @@ version: '2.1'
 
 services:
   elastic:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1
+    # arm64v8 is for Apple M1
+    image: webhippie/elasticsearch:6.4
     ports:
       - 9200:9200
       - 9300:9300
@@ -23,31 +24,41 @@ services:
       interval: 10s
       timeout: 10s
       retries: 10
-  db:
-    image: postgres
-    restart: always
-    ports:
-    - "5432:5432"
-    environment:
-      POSTGRES_USER: "pguser"
-      POSTGRES_PASSWORD: "pgpass"
-      POSTGRES_DB: "pgdb"
+    volumes:
+      # for normal elasticsearch:
+      # - elasticsearch:/usr/share/elasticsearch/data
+      # for webhippie/elasticsearch:
+      - elasticsearch:/var/lib/elasticsearch
 
-# uncomment to get kibana, which can be useful for analyzing index contents
-# this is currently not a required part of testing infrastructure, just left here for convenience
-#  kibana:
-#    image: docker.elastic.co/kibana/kibana:6.0.1
-#    depends_on:
-#      elastic:
-#        condition: service_healthy
-#    links:
-#      - elastic
+  # commented out because right now we're not using postgres for tests
+# comment back in if we need it for multiprocessing tests
+#  db:
+#    image: postgres
+#    restart: always
 #    ports:
-#      - 5601:5601
+#    - "5432:5432"
 #    environment:
-#      - "ELASTICSEARCH_URL=http://elastic:9200"
-#    healthcheck:
-#        test: ["CMD", "curl", "-f", "http://localhost:5601"]
-#        interval: 10s
-#        timeout: 10s
-#        retries: 10
+#      POSTGRES_USER: "pguser"
+#      POSTGRES_PASSWORD: "pgpass"
+#      POSTGRES_DB: "pgdb"
+
+# uncomment to get kibana, which can be useful for analyzing index contents
+# this is currently not a required part of testing infrastructure, just left here for convenience
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.4.3
+    depends_on:
+      elastic:
+        condition: service_healthy
+    links:
+      - elastic
+    ports:
+      - 5601:5601
+    environment:
+      - "ELASTICSEARCH_URL=http://elastic:9200"
+    healthcheck:
+        test: ["CMD", "curl", "-f", "http://localhost:5601"]
+        interval: 10s
+        timeout: 10s
+        retries: 10
+volumes:
+    elasticsearch:

requirements/base.in

@@ -1,6 +1,6 @@
 # Core requirements for using this application
 
 Django                    # Web application framework
-elasticsearch-dsl==6.1.0  # For interacting with Elaticsearch
+elasticsearch-dsl<7.0.0  # For interacting with Elaticsearch
 texttable                 # Printing tabular data from management commands
 multiprocessing-logging   # for logging in ./manage.py es_update --workers

requirements/dev.txt

@@ -6,11 +6,11 @@
 #
 asgiref==3.7.2
     # via django
-astroid==2.15.5
+astroid==2.15.6
     # via
     #   pylint
     #   pylint-celery
-backports-functools-lru-cache==1.6.5
+backports-functools-lru-cache==1.6.6
     # via caniusepython3
 bleach==6.0.0
     # via readme-renderer
@@ -22,9 +22,9 @@ certifi==2023.5.7
     # via requests
 chardet==5.1.0
     # via diff-cover
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
-click==8.1.3
+click==8.1.4
     # via
     #   click-log
     #   code-annotations
@@ -42,7 +42,7 @@ distlib==0.3.6
     # via
     #   caniusepython3
     #   virtualenv
-django==4.2.2
+django==4.2.3
     # via -r requirements/base.in
 docutils==0.20.1
     # via readme-renderer
@@ -52,25 +52,23 @@ edx-lint==5.3.4
     #   -r requirements/quality.in
 elasticsearch==6.8.2
     # via elasticsearch-dsl
-elasticsearch-dsl==6.1.0
+elasticsearch-dsl==6.4.0
     # via -r requirements/base.in
 filelock==3.12.2
     # via
     #   tox
     #   virtualenv
 idna==3.4
     # via requests
-importlib-metadata==6.7.0
+importlib-metadata==6.8.0
     # via
     #   keyring
     #   twine
-ipaddress==1.0.23
-    # via elasticsearch-dsl
 isort==5.12.0
     # via
     #   -r requirements/quality.in
     #   pylint
-jaraco-classes==3.2.3
+jaraco-classes==3.3.0
     # via keyring
 jinja2==3.1.2
     # via
@@ -99,11 +97,11 @@ packaging==23.1
     #   tox
 pbr==5.11.1
     # via stevedore
-pip-tools==6.13.0
+pip-tools==6.14.0
     # via -r requirements/dev.in
 pkginfo==1.9.6
     # via twine
-platformdirs==3.8.0
+platformdirs==3.8.1
     # via
     #   pylint
     #   virtualenv
@@ -177,6 +175,7 @@ texttable==1.6.7
 tomli==2.0.1
     # via
     #   build
+    #   pip-tools
     #   pylint
     #   pyproject-hooks
     #   tox
@@ -193,7 +192,7 @@ tox-pyenv==1.1.0
     # via -r requirements/dev.in
 twine==4.0.2
     # via -r requirements/dev.in
-typing-extensions==4.7.0
+typing-extensions==4.7.1
     # via
     #   asgiref
     #   astroid
@@ -213,7 +212,7 @@ wheel==0.40.0
     #   pip-tools
 wrapt==1.15.0
     # via astroid
-zipp==3.15.0
+zipp==3.16.0
     # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/quality.txt

@@ -4,19 +4,19 @@
 #
 #    pip-compile --output-file=requirements/quality.txt requirements/quality.in
 #
-astroid==2.15.5
+astroid==2.15.6
     # via
     #   pylint
     #   pylint-celery
-backports-functools-lru-cache==1.6.5
+backports-functools-lru-cache==1.6.6
     # via caniusepython3
 caniusepython3==7.3.0
     # via -r requirements/quality.in
 certifi==2023.5.7
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
-click==8.1.3
+click==8.1.4
     # via
     #   click-log
     #   code-annotations
@@ -49,7 +49,7 @@ packaging==23.1
     # via caniusepython3
 pbr==5.11.1
     # via stevedore
-platformdirs==3.8.0
+platformdirs==3.8.1
     # via pylint
 pycodestyle==2.10.0
     # via -r requirements/quality.in
@@ -87,7 +87,7 @@ tomli==2.0.1
     # via pylint
 tomlkit==0.11.8
     # via pylint
-typing-extensions==4.7.0
+typing-extensions==4.7.1
     # via
     #   astroid
     #   pylint

requirements/test.txt

@@ -8,7 +8,7 @@ asgiref==3.7.2
     # via django
 certifi==2023.5.7
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
 codecov==2.1.13
     # via -r requirements/test.in
@@ -27,13 +27,11 @@ filelock==3.12.2
     #   virtualenv
 idna==3.4
     # via requests
-ipaddress==1.0.23
-    # via elasticsearch-dsl
 multiprocessing-logging==0.3.4
     # via -r requirements/base.in
 packaging==23.1
     # via tox
-platformdirs==3.8.0
+platformdirs==3.8.1
     # via virtualenv
 pluggy==1.2.0
     # via tox
@@ -60,7 +58,7 @@ tox==3.28.0
     #   tox-battery
 tox-battery==0.6.1
     # via -r requirements/test.in
-typing-extensions==4.7.0
+typing-extensions==4.7.1
     # via asgiref
 urllib3==2.0.3
     # via

requirements/travis.txt

@@ -6,7 +6,7 @@
 #
 certifi==2023.5.7
     # via requests
-charset-normalizer==3.1.0
+charset-normalizer==3.2.0
     # via requests
 codecov==2.1.13
     # via -r requirements/travis.in
@@ -22,7 +22,7 @@ idna==3.4
     # via requests
 packaging==23.1
     # via tox
-platformdirs==3.8.0
+platformdirs==3.8.1
     # via virtualenv
 pluggy==1.2.0
     # via tox