Automatically split Batch with more than 10k requests to multiple Batch requests. Added cascadeCreate parameter to Set item/user values.

Author: OndraFiedler

README.md

@@ -29,36 +29,37 @@ Or install it yourself as:
 require 'recombee_api_client'
 include RecombeeApiClient
 
-# Prepare some items and users 
-NUM = 100
-my_users = (1..NUM).map { |i| "user-#{i}" }
-my_items = (1..NUM).map { |i| "item-#{i}" }
+client = RecombeeClient.new('client-test', 'jGGQ6ZKa8rQ1zTAyxTc0EMn55YPF7FJLUtaMLhbsGxmvwxgTwXYqmUk5xVZFw98L')
 
-#Generate some random purchases of items by users
+# Generate some random purchases of items by users
+NUM = 100
 PROBABILITY_PURCHASED = 0.1
-my_purchases = []
-my_users.each do |user|
-  p = my_items.select { |_| rand(0.0..1.0) < PROBABILITY_PURCHASED }
-  p.each { |item| my_purchases.push('userId' => user, 'itemId' => item) }
+
+users = (1..NUM).map { |i| "user-#{i}" }
+items = (1..NUM).map { |i| "item-#{i}" }
+purchases = []
+
+users.each do |user_id|
+  purchased = items.select { |_| rand(0.0..1.0) < PROBABILITY_PURCHASED }
+  purchased.each { |item_id| purchases.push(
+
+                AddPurchase.new(user_id, item_id,'cascadeCreate' => true)
+                                                  # Use cascadeCreate to create the
+                                                  # yet non-existing users and items
+                )}
+
 end
 
-# Use Recombee recommender
-client = RecombeeClient.new('client-test', 'jGGQ6ZKa8rQ1zTAyxTc0EMn55YPF7FJLUtaMLhbsGxmvwxgTwXYqmUk5xVZFw98L')
 begin
-  # Send the data to Recombee, use Batch for faster processing
-  puts 'Send users'
-  client.send(Batch.new(my_users.map { |userId| AddUser.new(userId) }))
-  puts 'Send items'
-  client.send(Batch.new(my_items.map { |itemId| AddItem.new(itemId) }))
-  puts 'Send purchases'
-  client.send(Batch.new(my_purchases.map { |p| AddPurchase.new(p['userId'], p['itemId']) }))
+  # Send the data to Recombee, use Batch for faster processing of larger data
+  client.send(Batch.new(purchases))
 
   # Get recommendations for user 'user-25'
-  puts 'Recommend for a user'
-  recommended = client.send(UserBasedRecommendation.new('user-25', 5, 'rotationRate' => 0))
-  puts "Recommended items: #{recommended}"
+  recommended = client.send(UserBasedRecommendation.new('user-25', 5))
+  puts "Recommended items for user-25: #{recommended}"
 rescue APIError => e
   puts e
+  # Use fallback
 end
 ```
 
@@ -71,7 +72,8 @@ NUM = 100
 PROBABILITY_PURCHASED = 0.1
 
 client = RecombeeClient.new('client-test', 'jGGQ6ZKa8rQ1zTAyxTc0EMn55YPF7FJLUtaMLhbsGxmvwxgTwXYqmUk5xVZFw98L')
-client.send(ResetDatabase.new)
+client.send(ResetDatabase.new) # Clear everything from the database
+
 # We will use computers as items in this example
 # Computers have three properties 
 #   - price (floating point number)
@@ -82,6 +84,7 @@ client.send(ResetDatabase.new)
 client.send(AddItemProperty.new('price', 'double'))
 client.send(AddItemProperty.new('num-cores', 'int'))
 client.send(AddItemProperty.new('description', 'string'))
+client.send(AddItemProperty.new('time', 'timestamp'))
 
 # Prepare requests for setting a catalog of computers
 requests = (1..NUM).map do |i|
@@ -92,14 +95,18 @@ requests = (1..NUM).map do |i|
         'price' => rand(15000.0 .. 25000.0),
         'num-cores' => rand(1..8),
         'description' => 'Great computer',
-        '!cascadeCreate' => true # Use !cascadeCreate for creating item
+        'time' => DateTime.now
+      },
+      #optional parameters:
+      {
+        'cascadeCreate' => true  # Use cascadeCreate for creating item
                                  # with given itemId, if it doesn't exist
       }
     )
 end
 
 # Send catalog to the recommender system
-client.send(Batch.new(requests))
+puts client.send(Batch.new(requests))
 
 # Prepare some purchases of items by users
 requests = []

lib/recombee_api_client.rb

@@ -17,6 +17,8 @@ module RecombeeApiClient
   class RecombeeClient
     include HTTParty
 
+    BATCH_MAX_SIZE = 10000
+
     ##
     #   - +account+ -> Name of your account at Recombee
     #   - +token+ -> Secret token obtained from Recombee for signing requests
@@ -33,6 +35,9 @@ def initialize(account, token, protocol = 'http', options = {})
     ##
     #   - +request+ -> ApiRequest to be sent to Recombee recommender
     def send(request)
+
+      return send_multipart_batch(request) if request.kind_of? Batch and request.requests.size > BATCH_MAX_SIZE
+
       timeout = request.timeout / 1000
       uri = process_request_uri(request)
       uri = sign_url(uri)
@@ -94,6 +99,12 @@ def check_errors(response, request)
       fail ResponseError.new(request, status_code, response.body)
     end
 
+    def send_multipart_batch(request)
+      requests_parts = request.requests.each_slice(BATCH_MAX_SIZE)
+      responses = requests_parts.map {|rqs| Batch.new(rqs)}.map{|batch| send(batch)}
+      responses.inject([]){|result,resp| result + resp}
+    end
+
     def process_request_uri(request)
       uri = request.path
       uri.slice! ('/{databaseId}/')

lib/recombee_api_client/api/item_based_recommendation.rb

@@ -21,6 +21,12 @@ class ItemBasedRecommendation < ApiRequest
   #
   # * *Optional arguments (given as hash optional)*
   #   - +targetUserId+ -> ID of the user who will see the recommendations.
+  #
+  #Specifying the *targetUserId* is beneficial because:
+  #
+  #* It makes the recommendations personalized
+  #* Allows calculations of Actions and Conversions in the graphical user interface, as Recombee can pair the user who got recommendations and who afterwards viewed/purchased an item.
+  #
   #   - +userImpact+ -> If *targetUserId* parameter is present, the recommendations are biased towards the user given. Using *userImpact*, you may control this bias. For an extreme case of `userImpact=0.0`, the interactions made by the user are not taken into account at all (with the exception of history-based blacklisting), for `userImpact=1.0`, you'll get user-based recommendation. The default value is `0.1`
   #
   #   - +filter+ -> Boolean-returning [ReQL](https://docs.recombee.com/reql.html) expression which allows you to filter recommended items based on the values of their attributes.
@@ -70,11 +76,11 @@ class ItemBasedRecommendation < ApiRequest
   #
   #   - +diversity+ -> **Expert option** Real number from [0.0, 1.0] which determines how much mutually dissimilar should the recommended items be. The default value is 0.0, i.e., no diversification. Value 1.0 means maximal diversification.
   #
-  #   - +minRelevance+ -> **Expert option** Specifies the threshold of how much relevant must the recommended items be to the user. Possible values one of: "low", "medium", "high". The default value is "low", meaning that the system attempts to recommend number of items equal to *count* at any cost. If there are not enough data (such as interactions or item properties), this may even lead to bestseller-based recommendations to be appended to reach the full *count*. This behavior may be suppressed by using "medium" or "high" values. In such case, the system only recommends items of at least the requested qualit, and may return less than *count* items when there is not enough data to fulfill it.
+  #   - +minRelevance+ -> **Expert option** If the *targetUserId* is provided:  Specifies the threshold of how much relevant must the recommended items be to the user. Possible values one of: "low", "medium", "high". The default value is "low", meaning that the system attempts to recommend number of items equal to *count* at any cost. If there are not enough data (such as interactions or item properties), this may even lead to bestseller-based recommendations to be appended to reach the full *count*. This behavior may be suppressed by using "medium" or "high" values. In such case, the system only recommends items of at least the requested qualit, and may return less than *count* items when there is not enough data to fulfill it.
   #
-  #   - +rotationRate+ -> **Expert option** If your users browse the system in real-time, it may easily happen that you wish to offer them recommendations multiple times. Here comes the question: how much should the recommendations change? Should they remain the same, or should they rotate? Recombee API allows you to control this per-request in backward fashion. You may penalize an item for being recommended in the near past. For the specific user, `rotationRate=1` means maximal rotation, `rotationRate=0` means absolutely no rotation. You may also use, for example `rotationRate=0.2` for only slight rotation of recommended items.
+  #   - +rotationRate+ -> **Expert option** If the *targetUserId* is provided: If your users browse the system in real-time, it may easily happen that you wish to offer them recommendations multiple times. Here comes the question: how much should the recommendations change? Should they remain the same, or should they rotate? Recombee API allows you to control this per-request in backward fashion. You may penalize an item for being recommended in the near past. For the specific user, `rotationRate=1` means maximal rotation, `rotationRate=0` means absolutely no rotation. You may also use, for example `rotationRate=0.2` for only slight rotation of recommended items.
   #
-  #   - +rotationTime+ -> **Expert option** Taking *rotationRate* into account, specifies how long time it takes to an item to fully recover from the penalization. By example, `rotationTime=7200.0` means that items recommended more than 2 hours ago are definitely not penalized anymore. Currently, the penalization is linear, so for `rotationTime=7200.0`, an item is still penalized by `0.5` to the user after 1 hour.
+  #   - +rotationTime+ -> **Expert option** If the *targetUserId* is provided: Taking *rotationRate* into account, specifies how long time it takes to an item to fully recover from the penalization. For example, `rotationTime=7200.0` means that items recommended more than 2 hours ago are definitely not penalized anymore. Currently, the penalization is linear, so for `rotationTime=7200.0`, an item is still penalized by `0.5` to the user after 1 hour.
   #
   #
     def initialize(item_id, count, optional = {})

lib/recombee_api_client/api/set_item_values.rb

@@ -1,16 +1,12 @@
-#
-# This file is auto-generated, do not edit
-#
-
 module RecombeeApiClient
-  require_relative 'request'
+  require_relative 'set_values'
   require_relative '../errors'
 
   ##
   #Set/update (some) property values of a given item. The properties (columns) must be previously created by [Add item property](https://docs.recombee.com/api.html#add-item-property).
   #
-  class SetItemValues < ApiRequest
-    attr_reader :item_id, :values
+  class SetItemValues < SetValues
+    attr_reader :item_id
     attr_accessor :timeout
     attr_accessor :ensure_https
 
@@ -26,39 +22,24 @@ class SetItemValues < ApiRequest
   #    "product_description": "4K TV with 3D feature",
   #    "categories":   ["Electronics", "Televisions"],
   #    "price_usd": 342,
+  #    "in_stock_from": "2016-11-16T08:00Z",
   #    "!cascadeCreate": true
   #  }
   #```
   #
   #Special parameter `!cascadeCreate` may be used. It indicates that the item of the given itemId should be created if it does not exist in the database, as if the corresponding PUT method was used. Note the exclamation mark (!) at the beginning of the parameter's name to distinguish it from item property names.
   #
   #
-    def initialize(item_id, values)
+  # * *Optional arguments (given as hash optional)*
+  #   - +cascadeCreate+ -> Sets whether the item should be created  if not present in the database.
+  #
+    def initialize(item_id, values, optional = {})
+      super(values, optional)
       @item_id = item_id
-      @values = values
       @timeout = 1000
       @ensure_https = false
     end
 
-    # HTTP method
-    def method
-      :post
-    end
-  
-    # Values of body parameters as a Hash
-    def body_parameters
-      p = Hash.new
-      p = p.merge(@values)
-      p
-    end
-  
-    # Values of query parameters as a Hash.
-    # name of parameter => value of the parameter
-    def query_parameters
-      params = {}
-      params
-    end
-  
     # Relative path to the endpoint
     def path
       "/{databaseId}/items/#{@item_id}"

lib/recombee_api_client/api/set_user_values.rb

@@ -1,16 +1,12 @@
-#
-# This file is auto-generated, do not edit
-#
-
 module RecombeeApiClient
-  require_relative 'request'
+  require_relative 'set_values'
   require_relative '../errors'
 
   ##
   #Set/update (some) property values of a given user. The properties (columns) must be previously created by [Add user property](https://docs.recombee.com/api.html#add-user-property).
   #
-  class SetUserValues < ApiRequest
-    attr_reader :user_id, :values
+  class SetUserValues < SetValues
+    attr_reader :user_id
     attr_accessor :timeout
     attr_accessor :ensure_https
 
@@ -25,39 +21,19 @@ class SetUserValues < ApiRequest
   #  {
   #    "country": "US",
   #    "sex": "F",
-  #    "!cascadeCreate": true
   #  }
   #```
   #
-  #Special parameter `!cascadeCreate` may be used. It indicates that the user of the given userId should be created if it does not exist in the database, as if the corresponding PUT method was used. Note the exclamation mark (!) at the beginning of the parameter's name to distinguish it from user property names.
-  #
+  # * *Optional arguments (given as hash optional)*
+  #   - +cascadeCreate+ -> Sets whether the item should be created  if not present in the database.
   #
-    def initialize(user_id, values)
+    def initialize(user_id, values, optional = {})
+      super(values, optional)
       @user_id = user_id
-      @values = values
       @timeout = 1000
       @ensure_https = false
     end
-  
-    # HTTP method
-    def method
-      :post
-    end
-  
-    # Values of body parameters as a Hash
-    def body_parameters
-      p = Hash.new
-      p = p.merge(@values)
-      p
-    end
-  
-    # Values of query parameters as a Hash.
-    # name of parameter => value of the parameter
-    def query_parameters
-      params = {}
-      params
-    end
-  
+
     # Relative path to the endpoint
     def path
       "/{databaseId}/users/#{@user_id}"

lib/recombee_api_client/api/user_based_recommendation.rb

@@ -71,7 +71,7 @@ class UserBasedRecommendation < ApiRequest
   #
   #   - +rotationRate+ -> **Expert option** If your users browse the system in real-time, it may easily happen that you wish to offer them recommendations multiple times. Here comes the question: how much should the recommendations change? Should they remain the same, or should they rotate? Recombee API allows you to control this per-request in backward fashion. You may penalize an item for being recommended in the near past. For the specific user, `rotationRate=1` means maximal rotation, `rotationRate=0` means absolutely no rotation. You may also use, for example `rotationRate=0.2` for only slight rotation of recommended items.
   #
-  #   - +rotationTime+ -> **Expert option** Taking *rotationRate* into account, specifies how long time it takes to an item to fully recover from the penalization. By example, `rotationTime=7200.0` means that items recommended more than 2 hours ago are definitely not penalized anymore. Currently, the penalization is linear, so for `rotationTime=7200.0`, an item is still penalized by `0.5` to the user after 1 hour.
+  #   - +rotationTime+ -> **Expert option** Taking *rotationRate* into account, specifies how long time it takes to an item to fully recover from the penalization. For example, `rotationTime=7200.0` means that items recommended more than 2 hours ago are definitely not penalized anymore. Currently, the penalization is linear, so for `rotationTime=7200.0`, an item is still penalized by `0.5` to the user after 1 hour.
   #
   #
     def initialize(user_id, count, optional = {})

spec/api/batch_spec.rb

@@ -34,4 +34,14 @@
     expect(repl[9]['json'].sort).to match_array []
   end
 
+  it 'send large multi-part batch' do
+
+    NUM = 23648
+    reqs = (1..NUM).map{|i| AddItem.new("item-#{i}")}
+    repl = @client.send(Batch.new(reqs))
+    expect(repl.size).to eq(NUM)
+
+    repl.each{|r| expect(r['code']).to eq(201)}
+
+  end
 end

spec/api/set_values.rb

@@ -27,6 +27,11 @@
     resp = @client.send(req)
   end
 
+  it 'does not fail with cascadeCreate optional parameter' do
+    req = described_class.new('new_entity2',{'int_property' => 5,'str_property' => 'test'},{'cascadeCreate' => true})
+    resp = @client.send(req)
+  end
+
   it 'fails with nonexisting entity' do
     req = described_class.new('nonexisting',{'int_property' => 5})
     expect { @client.send(req) }.to raise_exception { |exception|