Merge #257

257: Add methods to add JSON, CSV and NDJSON document  r=curquiza a=thicolares

### Addresses the following items of #243:

- [x] addDocumentsJson(string docs, string primaryKey)
- [x] addDocumentsCsv(string docs, string primaryKey)
- [x] addDocumentsNdjson(string docs, string primaryKey)

### Changes

_Tip: in order to ease the code review, you can start reviewing it commit by commit._

- Expose request options as a parameter of `http_post`. Motivations:
    - Allow who knows about the content to use `transform_body?` in order to decide if `http_request` should or not transform the body to JSON;
    - Group `transform_body?`, `headers` and [other options' default values that were spread around](https://github.com/meilisearch/meilisearch-ruby/blob/1ddbc9253f15cc4879bd6e7392c145ae9af9bde9/lib/meilisearch/http_request.rb#L82-L83) into a single Hash;
    - It can be improved in the next iterations, but it will already ease the next tasks within #243.
- Create methods to add documents as JSON, NDJSON and CSV
    - Add related tests exposing the usage of primaryKeys
    - No _"add in batches"_ included in this PR.

Co-authored-by: Thiago Colares <[email protected]>
Co-authored-by: Clémentine Urquizar - curqui <[email protected]>

Author: 3 people

.rubocop_todo.yml

@@ -1,6 +1,6 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2021-11-19 14:59:32 UTC using RuboCop version 1.23.0.
+# on 2021-11-21 21:31:55 UTC using RuboCop version 1.23.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
@@ -14,6 +14,12 @@ Gemspec/RequireMFA:
   Exclude:
     - 'meilisearch.gemspec'
 
+# Offense count: 1
+# Cop supports --auto-correct.
+Layout/HeredocIndentation:
+  Exclude:
+    - 'spec/meilisearch/index/documents_spec.rb'
+
 # Offense count: 32
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
 # IgnoredMethods: refine
@@ -23,7 +29,12 @@ Metrics/BlockLength:
 # Offense count: 1
 # Configuration parameters: CountComments, CountAsOne.
 Metrics/ClassLength:
-  Max: 271
+  Max: 289
+
+# Offense count: 1
+# Configuration parameters: Max, CountKeywordArgs.
+Metrics/ParameterLists:
+  MaxOptionalParameters: 4
 
 # Offense count: 2
 Naming/AccessorMethodName:

lib/meilisearch/http_request.rb

@@ -12,32 +12,30 @@ class HTTPRequest
     def initialize(url, api_key = nil, options = {})
       @base_url = url
       @api_key = api_key
-      @options = options
-      @headers = {
-        'Content-Type' => 'application/json',
-        'X-Meili-API-Key' => api_key
-      }.compact
-      @headers_no_body = {
-        'X-Meili-API-Key' => api_key
-      }.compact
+      @options = merge_options({
+                                 timeout: 1,
+                                 max_retries: 0,
+                                 headers: build_default_options_headers(api_key),
+                                 convert_body?: true
+                               }, options)
     end
 
     def http_get(relative_path = '', query_params = {})
       send_request(
         proc { |path, config| self.class.get(path, config) },
         relative_path,
         query_params: query_params,
-        headers: @headers_no_body
+        options: remove_options_header(@options, 'Content-Type')
       )
     end
 
-    def http_post(relative_path = '', body = nil, query_params = nil)
+    def http_post(relative_path = '', body = nil, query_params = nil, options = {})
       send_request(
         proc { |path, config| self.class.post(path, config) },
         relative_path,
         query_params: query_params,
         body: body,
-        headers: @headers
+        options: merge_options(@options, options)
       )
     end
 
@@ -47,22 +45,48 @@ def http_put(relative_path = '', body = nil, query_params = nil)
         relative_path,
         query_params: query_params,
         body: body,
-        headers: @headers
+        options: @options
       )
     end
 
     def http_delete(relative_path = '')
       send_request(
         proc { |path, config| self.class.delete(path, config) },
         relative_path,
-        headers: @headers_no_body
+        options: remove_options_header(@options, 'Content-Type')
       )
     end
 
     private
 
-    def send_request(http_method, relative_path, query_params: nil, body: nil, headers: nil)
-      config = http_config(query_params, body, headers)
+    def build_default_options_headers(api_key = nil)
+      {
+        'Content-Type' => 'application/json',
+        'X-Meili-API-Key' => api_key
+      }.compact
+    end
+
+    def merge_options(default_options, added_options = {})
+      default_cloned_headers = default_options[:headers].clone
+      merged_options = default_options.merge(added_options)
+      merged_options[:headers] = default_cloned_headers.merge(added_options[:headers]) if added_options.key?(:headers)
+      merged_options
+    end
+
+    def remove_options_header(options, key)
+      cloned_options = clone_options(options)
+      cloned_options[:headers].tap { |headers| headers.delete(key) }
+      cloned_options
+    end
+
+    def clone_options(options)
+      cloned_options = options.clone
+      cloned_options[:headers] = options[:headers].clone
+      cloned_options
+    end
+
+    def send_request(http_method, relative_path, query_params: nil, body: nil, options: {})
+      config = http_config(query_params, body, options)
       begin
         response = http_method.call(@base_url + relative_path, config)
       rescue Errno::ECONNREFUSED => e
@@ -71,13 +95,14 @@ def send_request(http_method, relative_path, query_params: nil, body: nil, heade
       validate(response)
     end
 
-    def http_config(query_params, body, headers)
+    def http_config(query_params, body, options)
+      body = body.to_json if options[:convert_body?] == true
       {
-        headers: headers,
+        headers: options[:headers],
         query: query_params,
-        body: body.to_json,
-        timeout: @options[:timeout] || 1,
-        max_retries: @options[:max_retries] || 0
+        body: body,
+        timeout: options[:timeout],
+        max_retries: options[:max_retries]
       }.compact
     end
 

lib/meilisearch/index.rb

@@ -84,6 +84,27 @@ def add_documents!(documents, primary_key = nil)
     alias replace_documents! add_documents!
     alias add_or_replace_documents! add_documents!
 
+    def add_documents_json(documents, primary_key = nil)
+      options = { headers: { 'Content-Type' => 'application/json' }, convert_body?: false }
+      http_post "/indexes/#{@uid}/documents", documents, { primaryKey: primary_key }.compact, options
+    end
+    alias replace_documents_json add_documents_json
+    alias add_or_replace_documents_json add_documents_json
+
+    def add_documents_ndjson(documents, primary_key = nil)
+      options = { headers: { 'Content-Type' => 'application/x-ndjson' }, convert_body?: false }
+      http_post "/indexes/#{@uid}/documents", documents, { primaryKey: primary_key }.compact, options
+    end
+    alias replace_documents_ndjson add_documents_ndjson
+    alias add_or_replace_documents_ndjson add_documents_ndjson
+
+    def add_documents_csv(documents, primary_key = nil)
+      options = { headers: { 'Content-Type' => 'text/csv' }, convert_body?: false }
+      http_post "/indexes/#{@uid}/documents", documents, { primaryKey: primary_key }.compact, options
+    end
+    alias replace_documents_csv add_documents_csv
+    alias add_or_replace_documents_csv add_documents_csv
+
     def update_documents(documents, primary_key = nil)
       documents = [documents] if documents.is_a?(Hash)
       http_put "/indexes/#{@uid}/documents", documents, { primaryKey: primary_key }.compact

spec/meilisearch/index/base_spec.rb

@@ -87,7 +87,15 @@
     options = { timeout: 2, max_retries: 1 }
     new_client = MeiliSearch::Client.new(URL, MASTER_KEY, options)
     index = new_client.create_index('options')
-    expect(index.options).to eq({ timeout: 2, max_retries: 1 })
+    expect(index.options).to eq({
+                                  headers: {
+                                    'Content-Type' => 'application/json',
+                                    'X-Meili-API-Key' => MASTER_KEY
+                                  },
+                                  max_retries: 1,
+                                  timeout: 2,
+                                  convert_body?: true
+                                })
     expect(MeiliSearch::Index).to receive(:get).with(
       "#{URL}/indexes/options",
       {

spec/meilisearch/index/documents_spec.rb

@@ -36,6 +36,52 @@
         expect(index.documents.first.keys).to eq(docs.first.keys.map(&:to_s))
       end
 
+      it 'adds JSON documents (as a array of documents)' do
+        documents = <<JSON
+        [
+          { "objectRef": 123,  "title": "Pride and Prejudice",                    "comment": "A great book" },
+          { "objectRef": 456,  "title": "Le Petit Prince",                        "comment": "A french book" },
+          { "objectRef": 1,    "title": "Alice In Wonderland",                    "comment": "A weird book" },
+          { "objectRef": 1344, "title": "The Hobbit",                             "comment": "An awesome book" },
+          { "objectRef": 4,    "title": "Harry Potter and the Half-Blood Prince", "comment": "The best book" }
+        ]
+JSON
+        response = index.add_documents_json(documents, 'objectRef')
+
+        expect(response).to be_a(Hash)
+        expect(response).to have_key('updateId')
+        index.wait_for_pending_update(response['updateId'])
+        expect(index.documents.count).to eq(5)
+      end
+
+      it 'adds NDJSON documents (as a array of documents)' do
+        documents = <<NDJSON
+        { "objectRef": 123,  "title": "Pride and Prejudice",                    "comment": "A great book" }
+        { "objectRef": 456,  "title": "Le Petit Prince",                        "comment": "A french book" }
+        { "objectRef": 1,    "title": "Alice In Wonderland",                    "comment": "A weird book" }
+        { "objectRef": 4,    "title": "Harry Potter and the Half-Blood Prince", "comment": "The best book" }
+NDJSON
+        response = index.add_documents_ndjson(documents, 'objectRef')
+        expect(response).to be_a(Hash)
+        expect(response).to have_key('updateId')
+        index.wait_for_pending_update(response['updateId'])
+        expect(index.documents.count).to eq(4)
+      end
+
+      it 'adds CSV documents (as a array of documents)' do
+        documents = <<CSV
+"objectRef:number","title:string","comment:string"
+"1239","Pride and Prejudice","A great book"
+"4569","Le Petit Prince","A french book"
+"49","Harry Potter and the Half-Blood Prince","The best book"
+CSV
+        response = index.add_documents_csv(documents, 'objectRef')
+        expect(response).to be_a(Hash)
+        expect(response).to have_key('updateId')
+        index.wait_for_pending_update(response['updateId'])
+        expect(index.documents.count).to eq(3)
+      end
+
       it 'adds documents in a batch (as a array of documents)' do
         response = index.add_documents_in_batches(documents, 5)
         expect(response).to be_a(Array)