Switch from elasticsearch-dsl-py to elasticsearch-py

This commit switches the dependency away from elasticsearch-dsl-py
as the upstream has been merged into elasticsearch-py. This commit
also updates the test matrix to support ES9

Author: sximba

README.rst

@@ -12,17 +12,17 @@ Django Elasticsearch DSL
     :target: https://django-elasticsearch-dsl.readthedocs.io/en/latest/
 
 Django Elasticsearch DSL is a package that allows indexing of django models in elasticsearch.
-It is built as a thin wrapper around elasticsearch-dsl-py_
-so you can use all the features developed by the elasticsearch-dsl-py team.
+It is built as a thin wrapper around elasticsearch-py_
+so you can use all the features developed by the elasticsearch-py team.
 
 You can view the full documentation at https://django-elasticsearch-dsl.readthedocs.io
 
-.. _elasticsearch-dsl-py: https://github.com/elastic/elasticsearch-dsl-py
+.. _elasticsearch-py: https://github.com/elastic/elasticsearch-py
 
 Features
 --------
 
-- Based on elasticsearch-dsl-py_ so you can make queries with the Search_ class.
+- Based on elasticsearch-py_ so you can make queries with the Search_ class.
 - Django signal receivers on save and delete for keeping Elasticsearch in sync.
 - Management commands for creating, deleting, rebuilding and populating indices.
 - Elasticsearch auto mapping from django models fields.
@@ -31,12 +31,14 @@ Features
 - Requirements
 
    - Django >= 3.2
-   - Python 3.8, 3.9, 3.10, 3.11
+   - Python 3.8, 3.9, 3.10, 3.11, 3.12, 3.13
 
 **Elasticsearch Compatibility:**
 The library is compatible with all Elasticsearch versions since 5.x
 **but you have to use a matching major version:**
 
+- For Elasticsearch 9.0 and later, use the major version 9 (9.x.y) of the library.
+
 - For Elasticsearch 8.0 and later, use the major version 8 (8.x.y) of the library.
 
 - For Elasticsearch 7.0 and later, use the major version 7 (7.x.y) of the library.
@@ -45,6 +47,9 @@ The library is compatible with all Elasticsearch versions since 5.x
 
 .. code-block:: python
 
+    # Elasticsearch 9.x
+    elasticsearch>=9.0.0,<10.0.0
+
     # Elasticsearch 8.x
     elasticsearch-dsl>=8.0.0,<9.0.0
 

django_elasticsearch_dsl/apps.py

@@ -2,11 +2,11 @@
 from django.conf import settings
 from django.utils.module_loading import import_string
 
-from elasticsearch_dsl.connections import connections
+from elasticsearch.dsl.connections import connections
 
 
 class DEDConfig(AppConfig):
-    name = 'django_elasticsearch_dsl'
+    name = "django_elasticsearch_dsl"
     verbose_name = "Django elasticsearch-dsl"
     signal_processor = None
 
@@ -17,20 +17,20 @@ def ready(self):
         if not self.signal_processor:
             signal_processor_path = getattr(
                 settings,
-                'ELASTICSEARCH_DSL_SIGNAL_PROCESSOR',
-                'django_elasticsearch_dsl.signals.RealTimeSignalProcessor'
+                "ELASTICSEARCH_DSL_SIGNAL_PROCESSOR",
+                "django_elasticsearch_dsl.signals.RealTimeSignalProcessor",
             )
             signal_processor_class = import_string(signal_processor_path)
             self.signal_processor = signal_processor_class(connections)
 
     @classmethod
     def autosync_enabled(cls):
-        return getattr(settings, 'ELASTICSEARCH_DSL_AUTOSYNC', True)
+        return getattr(settings, "ELASTICSEARCH_DSL_AUTOSYNC", True)
 
     @classmethod
     def default_index_settings(cls):
-        return getattr(settings, 'ELASTICSEARCH_DSL_INDEX_SETTINGS', {})
+        return getattr(settings, "ELASTICSEARCH_DSL_INDEX_SETTINGS", {})
 
     @classmethod
     def auto_refresh_enabled(cls):
-        return getattr(settings, 'ELASTICSEARCH_DSL_AUTO_REFRESH', True)
+        return getattr(settings, "ELASTICSEARCH_DSL_AUTO_REFRESH", True)

django_elasticsearch_dsl/documents.py

@@ -7,7 +7,7 @@
 from django import VERSION as DJANGO_VERSION
 from django.db import models
 from elasticsearch.helpers import bulk, parallel_bulk
-from elasticsearch_dsl import Document as DSLDocument
+from elasticsearch.dsl import Document as DSLDocument
 from six import iteritems
 
 from .exceptions import ModelFieldNotMappedError
@@ -21,7 +21,8 @@
     KeywordField,
     LongField,
     ShortField,
-    TextField, TimeField,
+    TextField,
+    TimeField,
 )
 from .search import Search
 from .signals import post_index
@@ -88,7 +89,7 @@ def search(cls, using=None, index=None):
             using=cls._get_using(using),
             index=cls._default_index(index),
             doc_type=[cls],
-            model=cls.django.model
+            model=cls.django.model,
         )
 
     def get_queryset(self):
@@ -104,7 +105,7 @@ def get_indexing_queryset(self):
         qs = self.get_queryset()
         kwargs = {}
         if DJANGO_VERSION >= (2,) and self.django.queryset_pagination:
-            kwargs = {'chunk_size': self.django.queryset_pagination}
+            kwargs = {"chunk_size": self.django.queryset_pagination}
         return qs.iterator(**kwargs)
 
     def init_prepare(self):
@@ -113,7 +114,7 @@ def init_prepare(self):
         from the model and generate a list of callables to avoid doing that
         work on every object instance over.
         """
-        index_fields = getattr(self, '_fields', {})
+        index_fields = getattr(self, "_fields", {})
         fields = []
         for name, field in iteritems(index_fields):
             if not isinstance(field, DEDField):
@@ -122,15 +123,20 @@ def init_prepare(self):
             if not field._path:
                 field._path = [name]
 
-            prep_func = getattr(self, 'prepare_%s_with_related' % name, None)
+            prep_func = getattr(self, "prepare_%s_with_related" % name, None)
             if prep_func:
-                fn = partial(prep_func, related_to_ignore=self._related_instance_to_ignore)
+                fn = partial(
+                    prep_func, related_to_ignore=self._related_instance_to_ignore
+                )
             else:
-                prep_func = getattr(self, 'prepare_%s' % name, None)
+                prep_func = getattr(self, "prepare_%s" % name, None)
                 if prep_func:
                     fn = prep_func
                 else:
-                    fn = partial(field.get_value_from_instance, field_value_to_ignore=self._related_instance_to_ignore)
+                    fn = partial(
+                        field.get_value_from_instance,
+                        field_value_to_ignore=self._related_instance_to_ignore,
+                    )
 
             fields.append((name, field, fn))
 
@@ -166,8 +172,9 @@ def to_field(cls, field_name, model_field):
         model field to ES field logic
         """
         try:
-            return cls.get_model_field_class_to_field_class()[
-                model_field.__class__](attr=field_name)
+            return cls.get_model_field_class_to_field_class()[model_field.__class__](
+                attr=field_name
+            )
         except KeyError:
             raise ModelFieldNotMappedError(
                 "Cannot convert model field {} "
@@ -178,17 +185,16 @@ def bulk(self, actions, **kwargs):
         response = bulk(client=self._get_connection(), actions=actions, **kwargs)
         # send post index signal
         post_index.send(
-            sender=self.__class__,
-            instance=self,
-            actions=actions,
-            response=response
+            sender=self.__class__, instance=self, actions=actions, response=response
         )
         return response
 
     def parallel_bulk(self, actions, **kwargs):
-        if self.django.queryset_pagination and 'chunk_size' not in kwargs:
-            kwargs['chunk_size'] = self.django.queryset_pagination
-        bulk_actions = parallel_bulk(client=self._get_connection(), actions=actions, **kwargs)
+        if self.django.queryset_pagination and "chunk_size" not in kwargs:
+            kwargs["chunk_size"] = self.django.queryset_pagination
+        bulk_actions = parallel_bulk(
+            client=self._get_connection(), actions=actions, **kwargs
+        )
         # As the `parallel_bulk` is lazy, we need to get it into `deque` to run it instantly
         # See https://discuss.elastic.co/t/helpers-parallel-bulk-in-python-not-working/39498/2
         deque(bulk_actions, maxlen=0)
@@ -207,29 +213,26 @@ def generate_id(cls, object_instance):
 
     def _prepare_action(self, object_instance, action):
         return {
-            '_op_type': action,
-            '_index': self._index._name,
-            '_id': self.generate_id(object_instance),
-            '_source': (
-                self.prepare(object_instance) if action != 'delete' else None
-            ),
+            "_op_type": action,
+            "_index": self._index._name,
+            "_id": self.generate_id(object_instance),
+            "_source": (self.prepare(object_instance) if action != "delete" else None),
         }
 
     def _get_actions(self, object_list, action):
         for object_instance in object_list:
-            if action == 'delete' or self.should_index_object(object_instance):
+            if action == "delete" or self.should_index_object(object_instance):
                 yield self._prepare_action(object_instance, action)
-    
+
     def get_actions(self, object_list, action):
         """
         Generate the elasticsearch payload.
         """
         return self._get_actions(object_list, action)
 
-
     def _bulk(self, *args, **kwargs):
         """Helper for switching between normal and parallel bulk operation"""
-        parallel = kwargs.pop('parallel', False)
+        parallel = kwargs.pop("parallel", False)
         if parallel:
             return self.parallel_bulk(*args, **kwargs)
         else:
@@ -242,24 +245,22 @@ def should_index_object(self, obj):
         """
         return True
 
-    def update(self, thing, refresh=None, action='index', parallel=False, **kwargs):
+    def update(self, thing, refresh=None, action="index", parallel=False, **kwargs):
         """
         Update each document in ES for a model, iterable of models or queryset
         """
         if refresh is not None:
-            kwargs['refresh'] = refresh
+            kwargs["refresh"] = refresh
         elif self.django.auto_refresh:
-            kwargs['refresh'] = self.django.auto_refresh
+            kwargs["refresh"] = self.django.auto_refresh
 
         if isinstance(thing, models.Model):
             object_list = [thing]
         else:
             object_list = thing
 
         return self._bulk(
-            self._get_actions(object_list, action),
-            parallel=parallel,
-            **kwargs
+            self._get_actions(object_list, action), parallel=parallel, **kwargs
         )
 
 

django_elasticsearch_dsl/fields.py

@@ -10,7 +10,7 @@
 else:
     from django.utils.encoding import force_str
 from django.utils.functional import Promise
-from elasticsearch_dsl.field import (
+from elasticsearch.dsl.field import (
     Boolean,
     Byte,
     Completion,
@@ -38,10 +38,10 @@
 class DEDField(Field):
     def __init__(self, attr=None, **kwargs):
         super(DEDField, self).__init__(**kwargs)
-        self._path = attr.split('.') if attr else []
+        self._path = attr.split(".") if attr else []
 
     def __setattr__(self, key, value):
-        if key == 'get_value_from_instance':
+        if key == "get_value_from_instance":
             self.__dict__[key] = value
         else:
             super(DEDField, self).__setattr__(key, value)
@@ -57,21 +57,15 @@ def get_value_from_instance(self, instance, field_value_to_ignore=None):
         for attr in self._path:
             try:
                 instance = instance[attr]
-            except (
-                TypeError, AttributeError,
-                KeyError, ValueError, IndexError
-            ):
+            except (TypeError, AttributeError, KeyError, ValueError, IndexError):
                 try:
                     instance = getattr(instance, attr)
                 except ObjectDoesNotExist:
                     return None
                 except (TypeError, AttributeError):
                     try:
                         instance = instance[int(attr)]
-                    except (
-                        IndexError, ValueError,
-                        KeyError, TypeError
-                    ):
+                    except (IndexError, ValueError, KeyError, TypeError):
                         if self._required:
                             raise VariableLookupError(
                                 "Failed lookup for key [{}] in "
@@ -100,29 +94,28 @@ class ObjectField(DEDField, Object):
     def _get_inner_field_data(self, obj, field_value_to_ignore=None):
         data = {}
 
-        if hasattr(self, 'properties'):
+        if hasattr(self, "properties"):
             for name, field in self.properties.to_dict().items():
                 if not isinstance(field, DEDField):
                     continue
 
                 if field._path == []:
                     field._path = [name]
 
-                data[name] = field.get_value_from_instance(
-                    obj, field_value_to_ignore
-                )
+                data[name] = field.get_value_from_instance(obj, field_value_to_ignore)
         else:
             doc_instance = self._doc_class()
             for name, field in self._doc_class._doc_type.mapping.properties._params.get(
-                'properties', {}).items():  # noqa
+                "properties", {}
+            ).items():  # noqa
                 if not isinstance(field, DEDField):
                     continue
 
                 if field._path == []:
                     field._path = [name]
 
                 # This allows for retrieving data from an InnerDoc with prepare_field_name functions.
-                prep_func = getattr(doc_instance, 'prepare_%s' % name, None)
+                prep_func = getattr(doc_instance, "prepare_%s" % name, None)
 
                 if prep_func:
                     data[name] = prep_func(obj)
@@ -155,7 +148,8 @@ def get_value_from_instance(self, instance, field_value_to_ignore=None):
         if is_iterable and not isinstance(objs, dict):
             return [
                 self._get_inner_field_data(obj, field_value_to_ignore)
-                for obj in objs if obj != field_value_to_ignore
+                for obj in objs
+                if obj != field_value_to_ignore
             ]
 
         return self._get_inner_field_data(objs, field_value_to_ignore)
@@ -248,11 +242,12 @@ class SearchAsYouTypeField(DEDField, SearchAsYouType):
 class FileFieldMixin(object):
     def get_value_from_instance(self, instance, field_value_to_ignore=None):
         _file = super(FileFieldMixin, self).get_value_from_instance(
-            instance, field_value_to_ignore)
+            instance, field_value_to_ignore
+        )
 
         if isinstance(_file, FieldFile):
-            return _file.url if _file else ''
-        return _file if _file else ''
+            return _file.url if _file else ""
+        return _file if _file else ""
 
 
 class FileField(FileFieldMixin, DEDField, Text):
@@ -261,8 +256,9 @@ class FileField(FileFieldMixin, DEDField, Text):
 
 class TimeField(KeywordField):
     def get_value_from_instance(self, instance, field_value_to_ignore=None):
-        time = super(TimeField, self).get_value_from_instance(instance,
-                                                              field_value_to_ignore)
+        time = super(TimeField, self).get_value_from_instance(
+            instance, field_value_to_ignore
+        )
 
         if time:
             return time.isoformat()

django_elasticsearch_dsl/indices.py

@@ -1,6 +1,6 @@
 from copy import deepcopy
 
-from elasticsearch_dsl import Index as DSLIndex
+from elasticsearch.dsl import Index as DSLIndex
 from six import python_2_unicode_compatible
 
 from .apps import DEDConfig