Merge #599

599: Support batching and updating with JSON, NDJSON, and CSV r=ellnix a=ellnix

# Pull Request

## Related issue
Fixes #243

The issue is very old, I added what I saw was missing in it, hopefully now we can close it. Reviewing can be done commit-by-commit, might be easier to follow.

Co-authored-by: ellnix <[email protected]>

Author: meili-bors[bot]

.rubocop_todo.yml

@@ -1,12 +1,12 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2025-01-15 12:21:42 UTC using RuboCop version 1.69.2.
+# on 2025-01-22 14:03:07 UTC using RuboCop version 1.69.2.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 67
+# Offense count: 70
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 # AllowedMethods: refine
 Metrics/BlockLength:
@@ -15,9 +15,9 @@ Metrics/BlockLength:
 # Offense count: 4
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 492
+  Max: 537
 
-# Offense count: 1
+# Offense count: 3
 # Configuration parameters: Max, CountKeywordArgs.
 Metrics/ParameterLists:
   MaxOptionalParameters: 4

lib/meilisearch/http_request.rb

@@ -22,14 +22,14 @@ def initialize(url, api_key = nil, options = {})
       @headers = build_default_options_headers
     end
 
-    def http_get(relative_path = '', query_params = {})
+    def http_get(relative_path = '', query_params = {}, options = {})
       send_request(
         proc { |path, config| self.class.get(path, config) },
         relative_path,
         config: {
           query_params: query_params,
-          headers: remove_headers(@headers.dup, 'Content-Type'),
-          options: @options
+          headers: remove_headers(@headers.dup.merge(options[:headers] || {}), 'Content-Type'),
+          options: @options.merge(options)
         }
       )
     end
@@ -47,40 +47,40 @@ def http_post(relative_path = '', body = nil, query_params = nil, options = {})
       )
     end
 
-    def http_put(relative_path = '', body = nil, query_params = nil)
+    def http_put(relative_path = '', body = nil, query_params = nil, options = {})
       send_request(
         proc { |path, config| self.class.put(path, config) },
         relative_path,
         config: {
           query_params: query_params,
           body: body,
-          headers: @headers,
-          options: @options
+          headers: @headers.dup.merge(options[:headers] || {}),
+          options: @options.merge(options)
         }
       )
     end
 
-    def http_patch(relative_path = '', body = nil, query_params = nil)
+    def http_patch(relative_path = '', body = nil, query_params = nil, options = {})
       send_request(
         proc { |path, config| self.class.patch(path, config) },
         relative_path,
         config: {
           query_params: query_params,
           body: body,
-          headers: @headers,
-          options: @options
+          headers: @headers.dup.merge(options[:headers] || {}),
+          options: @options.merge(options)
         }
       )
     end
 
-    def http_delete(relative_path = '', query_params = nil)
+    def http_delete(relative_path = '', query_params = nil, options = {})
       send_request(
         proc { |path, config| self.class.delete(path, config) },
         relative_path,
         config: {
           query_params: query_params,
-          headers: remove_headers(@headers.dup, 'Content-Type'),
-          options: @options
+          headers: remove_headers(@headers.dup.merge(options[:headers] || {}), 'Content-Type'),
+          options: @options.merge(options)
         }
       )
     end

lib/meilisearch/index.rb

@@ -145,6 +145,48 @@ def update_documents(documents, primary_key = nil)
     end
     alias add_or_update_documents update_documents
 
+    def update_documents_json(documents, primary_key = nil)
+      options = { convert_body?: false }
+      response = http_put "/indexes/#{@uid}/documents", documents, { primaryKey: primary_key }.compact, options
+
+      Models::Task.new(response, task_endpoint)
+    end
+    alias add_or_update_documents_json update_documents_json
+
+    def update_documents_ndjson(documents, primary_key = nil)
+      options = { headers: { 'Content-Type' => 'application/x-ndjson' }, convert_body?: false }
+      response = http_put "/indexes/#{@uid}/documents", documents, { primaryKey: primary_key }.compact, options
+
+      Models::Task.new(response, task_endpoint)
+    end
+    alias add_or_update_documents_ndjson update_documents_ndjson
+
+    def update_documents_csv(documents, primary_key = nil, delimiter = nil)
+      options = { headers: { 'Content-Type' => 'text/csv' }, convert_body?: false }
+
+      response = http_put "/indexes/#{@uid}/documents", documents, {
+        primaryKey: primary_key,
+        csvDelimiter: delimiter
+      }.compact, options
+
+      Models::Task.new(response, task_endpoint)
+    end
+    alias add_or_update_documents_csv add_documents_csv
+
+    def update_documents_ndjson_in_batches(documents, batch_size = 1000, primary_key = nil)
+      documents.lines.each_slice(batch_size).map do |batch|
+        update_documents_ndjson(batch.join, primary_key)
+      end
+    end
+
+    def update_documents_csv_in_batches(documents, batch_size = 1000, primary_key = nil, delimiter = nil)
+      lines = documents.lines
+      heading = lines.first
+      lines.drop(1).each_slice(batch_size).map do |batch|
+        update_documents_csv(heading + batch.join, primary_key, delimiter)
+      end
+    end
+
     def update_documents!(documents, primary_key = nil)
       Utils.soft_deprecate(
         'Index#update_documents!',
@@ -161,6 +203,20 @@ def add_documents_in_batches(documents, batch_size = 1000, primary_key = nil)
       end
     end
 
+    def add_documents_ndjson_in_batches(documents, batch_size = 1000, primary_key = nil)
+      documents.lines.each_slice(batch_size).map do |batch|
+        add_documents_ndjson(batch.join, primary_key)
+      end
+    end
+
+    def add_documents_csv_in_batches(documents, batch_size = 1000, primary_key = nil, delimiter = nil)
+      lines = documents.lines
+      heading = lines.first
+      lines.drop(1).each_slice(batch_size).map do |batch|
+        add_documents_csv(heading + batch.join, primary_key, delimiter)
+      end
+    end
+
     def add_documents_in_batches!(documents, batch_size = 1000, primary_key = nil)
       Utils.soft_deprecate(
         'Index#add_documents_in_batches!',