Merge #224

224: Added methods add_documents_in_batches, update_documents_in_batches and their tests r=curquiza a=FlamesRunner

This pull request adds the following methods:

- add_documents_in_batches(documents, batch_size = 1000, primary_key = nil)
- update_documents_in_batches(documents, batch_size = 1000, primary_key = nil)

The tests for these methods are:

- adds documents in a batch (as a array of documents)
- adds document batches synchronously (as an array of documents)
- updates documents in index in batches
- updates documents synchronously in index in batches (as an array of documents)

They are based on their non-batch methods. There are two linter errors:

```
lib/meilisearch/index.rb:7:3: C: Metrics/ClassLength: Class has too many lines. [252/226]
  class Index < HTTPRequest ...
  ^^^^^^^^^^^^^^^^^^^^^^^^^
spec/meilisearch/index/documents_spec.rb:3:1: C: Metrics/BlockLength: Block has too many lines. [500/497]
RSpec.describe 'MeiliSearch::Index - Documents' do ...
```

The linter indicates that there are too many lines for `documents_spec.rb` (which I was thinking of correcting by adding a new documents_batch_spec.rb, or the less clean way of changing the linter, etc), and that `index.rb` has too many lines (this one I am less sure of, as there aren't really any other classes that separate methods and since this is my first PR here, I don't want to introduce anything that might break something). I'm unsure, so I'd like to get some feedback on these.

References issue #218. 

Co-authored-by: Andrew Hong <[email protected]>
Co-authored-by: Andrew H <[email protected]>

Author: bors[bot]

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2021-09-01 13:50:39 UTC using RuboCop version 1.19.1.
+# on 2021-10-12 19:14:29 UTC using RuboCop version 1.20.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -14,16 +14,16 @@ Gemspec/DateAssignment:
   Exclude:
     - 'meilisearch.gemspec'
 
-# Offense count: 33
+# Offense count: 35
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
 # IgnoredMethods: refine
 Metrics/BlockLength:
-  Max: 497
+  Max: 500
 
 # Offense count: 1
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 226
+  Max: 256
 
 # Offense count: 1
 Naming/AccessorMethodName:

lib/meilisearch/index.rb

@@ -76,6 +76,40 @@ def update_documents!(documents, primary_key = nil)
     end
     alias add_or_update_documents! update_documents!
 
+    def add_documents_in_batches(documents, batch_size = 1000, primary_key = nil)
+      update_ids = []
+      documents.each_slice(batch_size) do |batch|
+        update_ids.append(add_documents(batch, primary_key))
+      end
+      update_ids
+    end
+
+    def add_documents_in_batches!(documents, batch_size = 1000, primary_key = nil)
+      update_ids = add_documents_in_batches(documents, batch_size, primary_key)
+      responses = []
+      update_ids.each do |update_object|
+        responses.append(wait_for_pending_update(update_object['updateId']))
+      end
+      responses
+    end
+
+    def update_documents_in_batches(documents, batch_size = 1000, primary_key = nil)
+      update_ids = []
+      documents.each_slice(batch_size) do |batch|
+        update_ids.append(update_documents(batch, primary_key))
+      end
+      update_ids
+    end
+
+    def update_documents_in_batches!(documents, batch_size = 1000, primary_key = nil)
+      update_ids = update_documents_in_batches(documents, batch_size, primary_key)
+      responses = []
+      update_ids.each do |update_object|
+        responses.append(wait_for_pending_update(update_object['updateId']))
+      end
+      responses
+    end
+
     def delete_documents(documents_ids)
       if documents_ids.is_a?(Array)
         http_post "/indexes/#{@uid}/documents/delete-batch", documents_ids

spec/meilisearch/index/documents_spec.rb

@@ -35,6 +35,17 @@
       expect(index.documents.count).to eq(documents.count)
     end
 
+    it 'adds documents in a batch (as a array of documents)' do
+      response = index.add_documents_in_batches(documents, 5)
+      expect(response).to be_a(Array)
+      expect(response.count).to eq(2) # 2 batches, since we start with 5 < documents.count <= 10 documents
+      expect(response[0]).to have_key('updateId')
+      response.each do |response_object|
+        index.wait_for_pending_update(response_object['updateId'])
+      end
+      expect(index.documents.count).to eq(documents.count)
+    end
+
     it 'adds documents synchronously (as an array of documents)' do
       response = index.add_documents!(documents)
       expect(response).to be_a(Hash)
@@ -45,6 +56,19 @@
       expect(index.documents.count).to eq(documents.count)
     end
 
+    it 'adds document batches synchronously (as an array of documents)' do
+      response = index.add_documents_in_batches!(documents, 5)
+      expect(response).to be_a(Array)
+      expect(response.count).to eq(2) # 2 batches, since we start with 5 < documents.count <= 10 documents
+      response.each do |response_object|
+        expect(response_object).to have_key('updateId')
+        expect(response_object).to have_key('status')
+        expect(response_object['status']).not_to eql('enqueued')
+        expect(response_object['status']).to eql('processed')
+      end
+      expect(index.documents.count).to eq(documents.count)
+    end
+
     it 'infers order of fields' do
       response = index.document(1)
       expect(response.keys).to eq(['objectId', 'title', 'comment'])
@@ -109,6 +133,31 @@
       expect(doc2['comment']).to eq(documents.detect { |doc| doc[:objectId] == id2 }[:comment])
     end
 
+    it 'updates documents in index in batches (as an array of documents)' do
+      id1 = 123
+      id2 = 456
+      updated_documents = [
+        { objectId: id1,  title: 'Sense and Sensibility' },
+        { objectId: id2,  title: 'The Little Prince' }
+      ]
+      response = index.update_documents_in_batches(updated_documents, 1)
+      expect(response).to be_a(Array)
+      expect(response.count).to eq(2)
+      response.each do |response_object|
+        expect(response_object).to have_key('updateId')
+        index.wait_for_pending_update(response_object['updateId'])
+      end
+
+      doc1 = index.document(id1)
+      doc2 = index.document(id2)
+
+      expect(index.documents.count).to eq(documents.count)
+      expect(doc1['title']).to eq(updated_documents.detect { |doc| doc[:objectId] == id1 }[:title])
+      expect(doc1['comment']).to eq(documents.detect { |doc| doc[:objectId] == id1 }[:comment])
+      expect(doc2['title']).to eq(updated_documents.detect { |doc| doc[:objectId] == id2 }[:title])
+      expect(doc2['comment']).to eq(documents.detect { |doc| doc[:objectId] == id2 }[:comment])
+    end
+
     it 'updates documents synchronously in index (as an array of documents)' do
       id1 = 123
       id2 = 456
@@ -131,6 +180,31 @@
       expect(doc2['comment']).to eq(documents.detect { |doc| doc[:objectId] == id2 }[:comment])
     end
 
+    it 'updates documents synchronously in index in batches (as an array of documents)' do
+      id1 = 123
+      id2 = 456
+      updated_documents = [
+        { objectId: id1,  title: 'Sense and Sensibility' },
+        { objectId: id2,  title: 'The Little Prince' }
+      ]
+      response = index.update_documents_in_batches!(updated_documents, 1)
+      expect(response).to be_a(Array)
+      expect(response.count).to eq(2) # 2 batches, since we have two items with batch size 1
+      response.each do |response_object|
+        expect(response_object).to have_key('updateId')
+        expect(response_object).to have_key('status')
+        expect(response_object['status']).not_to eql('enqueued')
+        expect(response_object['status']).to eql('processed')
+      end
+      doc1 = index.document(id1)
+      doc2 = index.document(id2)
+      expect(index.documents.count).to eq(documents.count)
+      expect(doc1['title']).to eq(updated_documents.detect { |doc| doc[:objectId] == id1 }[:title])
+      expect(doc1['comment']).to eq(documents.detect { |doc| doc[:objectId] == id1 }[:comment])
+      expect(doc2['title']).to eq(updated_documents.detect { |doc| doc[:objectId] == id2 }[:title])
+      expect(doc2['comment']).to eq(documents.detect { |doc| doc[:objectId] == id2 }[:comment])
+    end
+
     it 'updates one document in index (as an hash of one document)' do
       id = 123
       updated_document = { objectId: id, title: 'Emma' }