Added support for Elasticsearch 8 (#458)

* Updated Pillow version for compatibility with Python 3.8

* Updated alias actions to keyword arguments

* Updated elasticsearch-dsl version in requirements

* Updated tests

* Updated documentation

* Updated install requirements in setup.py

* Update runtests.py

* Update runtests.py

* Update runtests.py

* Update runtests.py

---------

Co-authored-by: Safwan Rahman <[email protected]>

Author: oehrlein

README.rst

@@ -37,12 +37,17 @@ Features
 The library is compatible with all Elasticsearch versions since 5.x
 **but you have to use a matching major version:**
 
+- For Elasticsearch 8.0 and later, use the major version 8 (8.x.y) of the library.
+
 - For Elasticsearch 7.0 and later, use the major version 7 (7.x.y) of the library.
 
 - For Elasticsearch 6.0 and later, use the major version 6 (6.x.y) of the library.
 
 .. code-block:: python
 
+    # Elasticsearch 8.x
+    elasticsearch-dsl>=8.0.0,<9.0.0
+
     # Elasticsearch 7.x
     elasticsearch-dsl>=7.0.0,<8.0.0
 

django_elasticsearch_dsl/management/commands/search_index.py

@@ -161,7 +161,7 @@ def _delete_alias_indices(self, alias):
         alias_delete_actions = [
             {"remove_index": {"index": index}} for index in alias_indices
         ]
-        self.es_conn.indices.update_aliases({"actions": alias_delete_actions})
+        self.es_conn.indices.update_aliases(actions=alias_delete_actions)
         for index in alias_indices:
             self.stdout.write("Deleted index '{}'".format(index))
 
@@ -231,7 +231,7 @@ def _update_alias(self, alias, new_index, alias_exists, options):
                 {"remove_index": {"index": index}} for index in old_indices
             ]
 
-        self.es_conn.indices.update_aliases({"actions": alias_actions})
+        self.es_conn.indices.update_aliases(actions=alias_actions)
         if delete_existing_index:
              self.stdout.write("Deleted index '{}'".format(alias))
 
@@ -247,7 +247,7 @@ def _update_alias(self, alias, new_index, alias_exists, options):
 
             if alias_delete_actions and not options['use_alias_keep_index']:
                 self.es_conn.indices.update_aliases(
-                    {"actions": alias_delete_actions}
+                    actions=alias_delete_actions
                 )
                 for index in old_indices:
                     self.stdout.write("Deleted index '{}'".format(index))

docs/source/quickstart.rst

@@ -20,8 +20,9 @@ For example:
 
     ELASTICSEARCH_DSL={
         'default': {
-            'hosts': 'localhost:9200'
-        },
+            'hosts': 'localhost:9200',
+            'http_auth': ('username', 'password')
+        }
     }
 
 ``ELASTICSEARCH_DSL`` is then passed to ``elasticsearch-dsl-py.connections.configure`` (see here_).

example/requirements.txt

@@ -4,5 +4,5 @@
 -e ../
 
 django-autofixture==0.12.1
-Pillow==6.2.0
+Pillow==6.2.2
 django==4.1.2

requirements.txt

@@ -1,2 +1,2 @@
 django>=3.2
-elasticsearch-dsl>=7.0.0,<8.0.0
+elasticsearch-dsl>=8.0.0,<9.0.0

runtests.py

@@ -9,6 +9,27 @@
     from django.test.utils import get_runner
 
     def get_settings(signal_processor):
+        elasticsearch_dsl_default_settings = {
+            'hosts': os.environ.get(
+                'ELASTICSEARCH_URL',
+                'https://127.0.0.1:9200'
+            ),
+            'basic_auth': (
+                os.environ.get('ELASTICSEARCH_USERNAME'),
+                os.environ.get('ELASTICSEARCH_PASSWORD')
+            )
+        }
+
+        elasticsearch_certs_path = os.environ.get(
+            'ELASTICSEARCH_CERTS_PATH'
+        )
+        if elasticsearch_certs_path:
+            elasticsearch_dsl_default_settings['ca_certs'] = (
+                elasticsearch_certs_path
+            )
+        else:
+            elasticsearch_dsl_default_settings['verify_certs'] = False
+ 
         PROCESSOR_CLASSES = {
             'realtime': 'django_elasticsearch_dsl.signals.RealTimeSignalProcessor',
             'celery': 'django_elasticsearch_dsl.signals.CelerySignalProcessor',
@@ -33,10 +54,7 @@ def get_settings(signal_processor):
             SITE_ID=1,
             MIDDLEWARE_CLASSES=(),
             ELASTICSEARCH_DSL={
-                'default': {
-                    'hosts': os.environ.get('ELASTICSEARCH_URL',
-                                            '127.0.0.1:9200')
-                },
+                'default': elasticsearch_dsl_default_settings
             },
             DEFAULT_AUTO_FIELD="django.db.models.BigAutoField",
             CELERY_BROKER_URL='memory://localhost/',
@@ -81,13 +99,42 @@ def make_parser():
         choices=('realtime', 'celery'),
         help='Defines which signal backend to choose'
     )
+    parser.add_argument(
+        '--elasticsearch-username',
+        nargs='?',
+        help="Username for Elasticsearch user"
+    )
+    parser.add_argument(
+        '--elasticsearch-password',
+        nargs='?',
+        help="Password for Elasticsearch user"
+    )
+    parser.add_argument(
+        '--elasticsearch-certs-path',
+        nargs='?',
+        help="Path to CA certificates for Elasticsearch"
+    )
     return parser
 
 
 def run_tests(*test_args):
     args, test_args = make_parser().parse_known_args(test_args)
     if args.elasticsearch:
-        os.environ.setdefault('ELASTICSEARCH_URL', args.elasticsearch)
+        os.environ.setdefault('ELASTICSEARCH_URL', "https://127.0.0.1:9200")
+
+        username = args.elasticsearch_username or "elastic"
+        password = args.elasticsearch_password or "changeme"
+        os.environ.setdefault(
+            'ELASTICSEARCH_USERNAME', username
+        )
+        os.environ.setdefault(
+            'ELASTICSEARCH_PASSWORD', password
+        )
+
+    if args.elasticsearch_certs_path:
+        os.environ.setdefault(
+            'ELASTICSEARCH_CERTS_PATH', args.elasticsearch_certs_path
+        )
 
     if not test_args:
         test_args = ['tests']

setup.py

@@ -42,7 +42,7 @@
     ],
     include_package_data=True,
     install_requires=[
-        'elasticsearch-dsl>=7.2.0,<8.0.0',
+        'elasticsearch-dsl>=8.9.0,<9.0.0',
         'six',
     ],
     license="Apache Software License 2.0",

tests/test_documents.py

@@ -508,8 +508,8 @@ def generate_id(cls, article):
 
         # Get the data from the elasticsearch low level API because
         # The generator get executed there.
-        data = json.loads(mock_bulk.call_args[1]['body'].split("\n")[0])
-        assert data["index"]["_id"] == article.slug
+        data = json.loads(mock_bulk.call_args[1]['operations'][1])
+        assert data['slug'] == article.slug
 
     @patch('elasticsearch_dsl.connections.Elasticsearch.bulk')
     def test_should_index_object_is_called(self, mock_bulk):
@@ -549,6 +549,13 @@ def should_index_object(self, obj):
 
         d = ArticleDocument()
         d.update([article1, article2])
+        operations = mock_bulk.call_args[1]['operations']
+        slugs = [
+            json.loads(operation)['slug'] for operation in operations
+            if 'slug' in json.loads(operation)
+        ]
+        self.assertTrue(article1.slug in slugs)
+        self.assertTrue(article2.slug not in slugs)
         data_body = mock_bulk.call_args[1]['body']
         self.assertTrue(article1.slug in data_body)
         self.assertTrue(article2.slug not in data_body)
@@ -558,4 +565,4 @@ class RealTimeDocTypeTestCase(BaseDocTypeTestCase, TestCase):
 
 
 class CeleryDocTypeTestCase(BaseDocTypeTestCase, TestCase):
-    TARGET_PROCESSOR = 'django_elasticsearch_dsl.signals.CelerySignalProcessor'
+    TARGET_PROCESSOR = 'django_elasticsearch_dsl.signals.CelerySignalProcessor'