Update elasticsearch client libraries to 8.x

.github/dependabot.yml

@@ -58,10 +58,10 @@ updates:
     - ">= 6"
   - dependency-name: elasticsearch
     versions:
-    - ">= 8"
-  - dependency-name: elasticsearch-dsl
+    - ">= 9"
+  - dependency-name: elastic-transport
     versions:
-    - ">= 8"
+    - ">= 9"
   - dependency-name: kombu
     versions:
     - ">= 6"

Dockerfile

@@ -18,7 +18,6 @@ ENV NPM_DEPS_DIR=${HOME}/node_modules
 ENV VITE_MANIFEST_FILE_NAME=manifest.json
 ENV STATIC_URL_PREFIX=/static-server/
 ENV MEDIA_URL_PREFIX=/user-media/
-ENV ELASTIC_CLIENT_APIVERSIONING=1
 RUN <<EOF
 groupadd -g ${OLYMPIA_UID} olympia
 useradd -u ${OLYMPIA_UID} -g ${OLYMPIA_UID} -s /sbin/nologin -d ${HOME} olympia

docker-compose.yml

@@ -6,7 +6,7 @@ x-env-mapping: &env
     - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia
     - CELERY_RESULT_BACKEND=redis://redis:6379/1
     - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia
-    - ELASTICSEARCH_LOCATION=elasticsearch:9200
+    - ELASTICSEARCH_LOCATION=http://elasticsearch:9200
     - MEMCACHE_LOCATION=memcached:11211
     - MYSQL_DATABASE=olympia
     - MYSQL_ROOT_PASSWORD=docker
@@ -21,7 +21,6 @@ x-env-mapping: &env
     - HISTCONTROL=erasedups
     - ENV=local
     - SWAGGER_UI_ENABLED=true
-    - CIRCLECI
     - SKIP_DATA_SEED
     - FXA_CLIENT_ID
     - FXA_CLIENT_SECRET

requirements/dev.txt

@@ -337,9 +337,6 @@ ruff==0.14.14 \
     --hash=sha256:e651e977a79e4c758eb807f0481d673a67ffe53cfa92209781dfa3a996cf8412 \
     --hash=sha256:e8058d2145566510790eab4e2fad186002e288dec5e0d343a92fe7b0bc1b3e13 \
     --hash=sha256:f666445819d31210b71e0a6d1c01e24447a20b85458eea25a25fe8142210ae0e
-typing_extensions==4.15.0 \
-    --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \
-    --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548
 hashin==1.0.5 \
     --hash=sha256:33f74f0e033cfc928dce246ccb1de8e38b45916b313678137306cccbfeeee47f \
     --hash=sha256:d1b9985addd3554ec85fd5b4e749b6faf577d49ac865a4ebda591ebf1e8f4d8a

requirements/prod.txt

@@ -167,13 +167,13 @@ pyparsing==3.3.2 \
 setuptools==80.9.0 \
     --hash=sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922 \
     --hash=sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c
-# six is required by Sphinx, bleach, cryptography, django-environ, django-extensions, django-tables2, docker-compose, docker-py, docker-pycreds, dockerpty, elasticsearch-dsl, html5lib, mock, nobot, packaging, pathlib2, prompt_toolkit, sphinxcontrib-httpdomain, websocket-client
+# six is required by Sphinx, bleach, cryptography, django-environ, django-extensions, django-tables2, docker-compose, docker-py, docker-pycreds, dockerpty, html5lib, mock, nobot, packaging, pathlib2, prompt_toolkit, sphinxcontrib-httpdomain, websocket-client
 six==1.17.0 \
     --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \
     --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81
-urllib3==1.26.20 \
-    --hash=sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e \
-    --hash=sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32
+urllib3==2.6.3 \
+    --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \
+    --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4
 # babel is required by Sphinx, django-babel, puente
 babel==2.17.0 \
     --hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \
@@ -812,13 +812,12 @@ dockerflow==2024.4.2 \
 drf-nested-routers==0.95.0 \
     --hash=sha256:815978f802e578fd7035c74040c104909cbe97615de89a275d77e928f4029891 \
     --hash=sha256:dd489c33d667aaa81383ffaa8c74781d2b353d8f0795716ae37fc59ee297b7c4
-# elasticsearch is required by elasticsearch-dsl
-elasticsearch==7.17.13 \
-    --hash=sha256:ba1b87322045fc37831009c7c49ef0e2777165a2ad3390df4d9e2b7a09d90c86 \
-    --hash=sha256:dac73df9a9070b9c96be45d99f4316badc525f8b17f418e9c86da2bb5855c892
-elasticsearch-dsl==7.4.1 \
-    --hash=sha256:07ee9c87dc28cc3cae2daa19401e1e18a172174ad9e5ca67938f752e3902a1d5 \
-    --hash=sha256:97f79239a252be7c4cce554c29e64695d7ef6a4828372316a5e5ff815e7a7498
+elasticsearch==8.19.3 \
+    --hash=sha256:e84dd618a220cac25b962790085045dd27ac72e01c0a5d81bd29a2d47a71f03f \
+    --hash=sha256:fe1db2555811192e8a1be78b01234d0a49d32b185ea7eeeb6f059331dee32838
+elastic-transport==8.17.1 \
+    --hash=sha256:192718f498f1d10c5e9aa8b9cf32aed405e469a7f0e9d6a8923431dbb2c59fb8 \
+    --hash=sha256:5edef32ac864dca8e2f0a613ef63491ee8d6b8cfb52881fa7313ba9290cac6d2
 email-reply-parser==0.5.12 \
     --hash=sha256:d051cfa8f54046f7399553aae57f17f62bf1652f83e3e9970fd109c0f24a880c \
     --hash=sha256:3499c02284679e020acf8aa30ef9e43c62f9ab5ccee0a35bfac85a0c1fa685fd \
@@ -970,7 +969,7 @@ pymemcache==4.0.0 \
 prompt_toolkit==3.0.52 \
     --hash=sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855 \
     --hash=sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955
-# python-dateutil is required by elasticsearch-dsl
+# python-dateutil is required by elasticsearch
 python-dateutil==2.9.0.post0 \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
@@ -1386,6 +1385,9 @@ pure_eval==0.2.3 \
 traitlets==5.14.3 \
     --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \
     --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f
+typing_extensions==4.15.0 \
+    --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \
+    --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548
 # Dependencies for IPython
 parso==0.8.5 \
     --hash=sha256:034d7354a9a018bdce352f48b2a8a450f05e9d6ee85db84764e9b6bd96dafe5a \

src/olympia/addons/tasks.py

@@ -8,7 +8,7 @@
 from django.db.models import Q, Value
 from django.db.models.functions import Collate
 
-from elasticsearch_dsl import Search
+from elasticsearch.dsl import Search
 
 import olympia.core
 from olympia import activity, amo

src/olympia/addons/views.py

@@ -10,7 +10,7 @@
 
 import waffle
 from drf_spectacular.utils import extend_schema, extend_schema_view
-from elasticsearch_dsl import Q, Search, query
+from elasticsearch.dsl import Q, Search, query
 from rest_framework import exceptions, serializers, status
 from rest_framework.decorators import action
 from rest_framework.generics import ListAPIView

src/olympia/api/serializers.py

@@ -5,7 +5,7 @@
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 
-from elasticsearch_dsl.response.hit import Hit
+from elasticsearch.dsl.response.hit import Hit
 from rest_framework import serializers
 
 from olympia import amo

src/olympia/lib/settings_base.py

@@ -928,7 +928,7 @@ def get_language_url_map():
             'level': logging.WARNING,
             'propagate': True,
         },
-        'elasticsearch': {
+        'elastic_transport.transport': {
             'handlers': ['null'],
             'level': logging.INFO,
             'propagate': False,

src/olympia/ratings/tests/test_models.py

@@ -3,7 +3,7 @@
 from django.core import mail
 from django.core.exceptions import ValidationError
 
-from elasticsearch_dsl import Q, Search
+from elasticsearch.dsl import Q, Search
 
 from olympia import amo, core
 from olympia.activity.models import ActivityLog

src/olympia/search/filters.py

@@ -7,7 +7,7 @@
 from django.utils.translation import gettext
 
 import colorgram
-from elasticsearch_dsl import Q, query
+from elasticsearch.dsl import Q, query
 from rest_framework import serializers
 from rest_framework.filters import BaseFilterBackend
 from waffle import switch_is_active

src/olympia/search/management/commands/reindex.py

@@ -268,7 +268,6 @@ def execute_workflow(self, workflow):
         self.stdout.write('\n')
 
         # Let's return the /_aliases values.
-        aliases = ES.indices.get_alias()
-        aliases = json.dumps(aliases, sort_keys=True, indent=4)
+        aliases = json.dumps(dict(ES.indices.get_alias()), sort_keys=True, indent=4)
         summary = _SUMMARY % aliases
         self.stdout.write(summary)

src/olympia/search/tests/test_filters.py

@@ -8,7 +8,7 @@
 from django.utils.http import urlsafe_base64_encode
 
 import time_machine
-from elasticsearch_dsl import Search
+from elasticsearch.dsl import Search
 from rest_framework import serializers
 
 from olympia import amo
@@ -179,7 +179,7 @@ def _test_q(self, qs, query):
                 'bool': {
                     'must': [
                         {'term': {'is_experimental': False}},
-                        {'terms': {'status': (4,)}},
+                        {'terms': {'status': [4]}},
                         {'exists': {'field': 'current_version'}},
                         {'term': {'is_disabled': False}},
                     ]
@@ -425,7 +425,7 @@ def test_status(self):
         assert 'must' not in qs['query']['bool']
         filter_ = qs['query']['bool']['filter']
 
-        assert {'terms': {'status': amo.APPROVED_STATUSES}} in filter_
+        assert {'terms': {'status': list(amo.APPROVED_STATUSES)}} in filter_
         assert {'exists': {'field': 'current_version'}} in filter_
         assert {'term': {'is_disabled': False}} in filter_
 
@@ -1527,7 +1527,7 @@ def test_combined(self):
         assert 'must_not' not in bool_
 
         filter_ = bool_['filter']
-        assert {'terms': {'status': amo.APPROVED_STATUSES}} in filter_
+        assert {'terms': {'status': list(amo.APPROVED_STATUSES)}} in filter_
         assert {'exists': {'field': 'current_version'}} in filter_
         assert {'term': {'is_disabled': False}} in filter_
 
@@ -1554,7 +1554,7 @@ def test_filter_featured_sort_random(self):
         assert 'must_not' not in bool_
 
         filter_ = bool_['filter']
-        assert {'terms': {'status': amo.APPROVED_STATUSES}} in filter_
+        assert {'terms': {'status': list(amo.APPROVED_STATUSES)}} in filter_
         assert {'exists': {'field': 'current_version'}} in filter_
         assert {'term': {'is_disabled': False}} in filter_
 
@@ -1572,7 +1572,7 @@ def test_filter_promoted_sort_random(self):
         assert 'must_not' not in bool_
 
         filter_ = bool_['filter']
-        assert {'terms': {'status': amo.APPROVED_STATUSES}} in filter_
+        assert {'terms': {'status': list(amo.APPROVED_STATUSES)}} in filter_
         assert {'exists': {'field': 'current_version'}} in filter_
         assert {'term': {'is_disabled': False}} in filter_
 

src/olympia/search/utils.py

@@ -5,7 +5,8 @@
 from django.conf import settings
 from django.core.management.base import CommandError
 
-from elasticsearch import Elasticsearch, TransportError, helpers, transport
+from elasticsearch import Elasticsearch, helpers, transport
+from elasticsearch.exceptions import NotFoundError
 
 from .models import Reindexing
 
@@ -74,7 +75,7 @@ def unindex_objects(ids, indexer_class):
     for id_ in ids:
         try:
             es.delete(index=index, id=id_)
-        except TransportError:
+        except NotFoundError:
             # We ignore already deleted objects.
             pass