Improved support for refresh during updates (#323)

* Increase configurability of bulk refresh parameter

The Elasticsearch bulk APIs support a configurable 'refresh' parameter
that can be set to:

1. empty string / true : perform refresh after operation completes
2. 'wait_for': wait until next scheduled refresh
3. false (default): don't do any refresh

See documentation at:

https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-refresh.html

Currently the `Document.update()` API accepts a `refresh` parameter
that, if `True`, passes `True` to the ES bulk API. There isn't currently
a way to pass 'wait_for'.

This commit modifies how `Document.update()` works so that it is
possible to pass `refresh='wait_for'`. I've implemented this in a
generic way, as opposed to hardcoding logic for 'wait_for', to support
future changes to this ES bulk API.

Note that, in keeping with the current convention, if
`refresh='wait_for'` is passed, this will override the value of
auto_refresh set either via setting or via the Document class. This
gives developers the maximum configurability here.

* Add support for --refresh to search_index command

Currently when updating or rebuilding an index, the `refresh` parameter
is never set, regardless if settings.ELASTICSEARCH_DSL_AUTO_REFRESH is
set or if auto_refresh is set on the Document class.

This commit adds the ability to pass `--refresh` to the `search_index`
management command, to force a refresh when the index is updated.

This can be useful, for example, if you want to programmatically call
that command from some Django tests that fill an index, and then
immediately run tests against that index. Without setting
`refresh=True`, there's no guarantee that the refreshed index will be
available to search.

* Update refresh parameter overrides auto_refresh

* Support Document.auto_refresh = 'wait_for'

Author: chosak

django_elasticsearch_dsl/documents.py

@@ -191,10 +191,10 @@ def update(self, thing, refresh=None, action='index', parallel=False, **kwargs):
         """
         Update each document in ES for a model, iterable of models or queryset
         """
-        if refresh is True or (
-            refresh is None and self.django.auto_refresh
-        ):
-            kwargs['refresh'] = True
+        if refresh is not None:
+            kwargs['refresh'] = refresh
+        elif self.django.auto_refresh:
+            kwargs['refresh'] = self.django.auto_refresh
 
         if isinstance(thing, models.Model):
             object_list = [thing]

django_elasticsearch_dsl/management/commands/search_index.py

@@ -64,6 +64,13 @@ def add_arguments(self, parser):
             help='Run populate/rebuild update single threaded'
         )
         parser.set_defaults(parallel=getattr(settings, 'ELASTICSEARCH_DSL_PARALLEL', False))
+        parser.add_argument(
+            '--refresh',
+            action='store_true',
+            dest='refresh',
+            default=None,
+            help='Refresh indices after populate/rebuild'
+        )
         parser.add_argument(
             '--no-count',
             action='store_false',
@@ -114,7 +121,7 @@ def _populate(self, models, options):
                 "(parallel)" if parallel else "")
             )
             qs = doc().get_indexing_queryset()
-            doc().update(qs, parallel=parallel)
+            doc().update(qs, parallel=parallel, refresh=options['refresh'])
 
     def _delete(self, models, options):
         index_names = [index._name for index in registry.get_indices(models)]

docs/source/management.rst

@@ -18,11 +18,11 @@ Populate the Elasticsearch mappings with the django models data (index need to b
 
 ::
 
-    $ search_index --populate [--models [app[.model] app[.model] ...]] [--parallel]
+    $ search_index --populate [--models [app[.model] app[.model] ...]] [--parallel] [--refresh]
 
 Recreate and repopulate the indices:
 
 ::
 
-    $ search_index --rebuild [-f] [--models [app[.model] app[.model] ...]] [--parallel]
+    $ search_index --rebuild [-f] [--models [app[.model] app[.model] ...]] [--parallel] [--refresh]
 

docs/source/quickstart.rst

@@ -87,7 +87,10 @@ It is required to define ``Document`` class in  ``documents.py`` in your app dir
             # or deleted:
             # ignore_signals = True
 
-            # Don't perform an index refresh after every update (overrides global setting):
+            # Configure how the index should be refreshed after an update.
+            # See Elasticsearch documentation for supported options:
+            # https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-refresh.html
+            # This per-Document setting overrides settings.ELASTICSEARCH_DSL_AUTO_REFRESH.
             # auto_refresh = False
 
             # Paginate the django queryset used to populate the index with the specified size

tests/test_commands.py

@@ -112,15 +112,24 @@ def test_create_all_indices(self):
 
     def test_populate_all_doc_type(self):
         call_command('search_index', stdout=self.out, action='populate')
+        expected_kwargs = {'parallel': False, 'refresh': None}
         # One call for "Indexing NNN documents", one for indexing itself (via get_index_queryset).
         assert self.doc_a1.get_queryset.call_count == 2
-        self.doc_a1.update.assert_called_once_with(self.doc_a1_qs.iterator(), parallel=False)
+        self.doc_a1.update.assert_called_once_with(self.doc_a1_qs.iterator(), **expected_kwargs)
         assert self.doc_a2.get_queryset.call_count == 2
-        self.doc_a2.update.assert_called_once_with(self.doc_a2_qs.iterator(), parallel=False)
+        self.doc_a2.update.assert_called_once_with(self.doc_a2_qs.iterator(), **expected_kwargs)
         assert self.doc_b1.get_queryset.call_count == 2
-        self.doc_b1.update.assert_called_once_with(self.doc_b1_qs.iterator(), parallel=False)
+        self.doc_b1.update.assert_called_once_with(self.doc_b1_qs.iterator(), **expected_kwargs)
         assert self.doc_c1.get_queryset.call_count == 2
-        self.doc_c1.update.assert_called_once_with(self.doc_c1_qs.iterator(), parallel=False)
+        self.doc_c1.update.assert_called_once_with(self.doc_c1_qs.iterator(), **expected_kwargs)
+
+    def test_populate_all_doc_type_refresh(self):
+        call_command('search_index', stdout=self.out, action='populate', refresh=True)
+        expected_kwargs = {'parallel': False, 'refresh': True}
+        self.doc_a1.update.assert_called_once_with(self.doc_a1_qs.iterator(), **expected_kwargs)
+        self.doc_a2.update.assert_called_once_with(self.doc_a2_qs.iterator(), **expected_kwargs)
+        self.doc_b1.update.assert_called_once_with(self.doc_b1_qs.iterator(), **expected_kwargs)
+        self.doc_c1.update.assert_called_once_with(self.doc_c1_qs.iterator(), **expected_kwargs)
 
     def test_rebuild_indices(self):
 

tests/test_documents.py

@@ -268,6 +268,46 @@ def test_model_instance_update_no_refresh(self):
             doc.update(car)
             self.assertNotIn('refresh', mock.call_args_list[0][1])
 
+    def test_model_instance_update_refresh_true(self):
+        doc = CarDocument()
+        doc.django.auto_refresh = False
+        car = Car()
+        with patch('django_elasticsearch_dsl.documents.bulk') as mock:
+            doc.update(car, refresh=True)
+            self.assertEqual(
+                mock.call_args_list[0][1]['refresh'], True
+            )
+
+    def test_model_instance_update_refresh_wait_for(self):
+        doc = CarDocument()
+        doc.django.auto_refresh = False
+        car = Car()
+        with patch('django_elasticsearch_dsl.documents.bulk') as mock:
+            doc.update(car, refresh='wait_for')
+            self.assertEqual(
+                mock.call_args_list[0][1]['refresh'], 'wait_for'
+            )
+
+    def test_model_instance_update_auto_refresh_wait_for(self):
+        doc = CarDocument()
+        doc.django.auto_refresh = 'wait_for'
+        car = Car()
+        with patch('django_elasticsearch_dsl.documents.bulk') as mock:
+            doc.update(car)
+            self.assertEqual(
+                mock.call_args_list[0][1]['refresh'], 'wait_for'
+            )
+
+    def test_model_instance_update_refresh_overrides_auto_refresh(self):
+        doc = CarDocument()
+        doc.django.auto_refresh = True
+        car = Car()
+        with patch('django_elasticsearch_dsl.documents.bulk') as mock:
+            doc.update(car, refresh=False)
+            self.assertEqual(
+                mock.call_args_list[0][1]['refresh'], False
+            )
+
     def test_model_instance_iterable_update_with_pagination(self):
         class CarDocument2(DocType):
             class Django: