Reverse pagination (#41)

Author: dokshina

CHANGELOG.md

@@ -23,6 +23,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 
 * `checks` is disabled for internal functions by default
+* `limit` option is renamed to `first`
+* Reverse pagination (negative `first`) is supported
 
 ## [0.1.0] - 2020-09-23
 

README.md

@@ -222,7 +222,9 @@ where:
 * `space_name` (`string`) - name of the space
 * `conditions` (`?table`) - array of [select conditions](#select-conditions)
 * `opts`:
-  * `limit` (`?number`) - the maximum limit of the objects to return
+  * `first` (`?number`) - the maximum count of the objects to return.
+     If negative value is specified, the last objects are returned
+     (`after` option is required in this case).
   * `after` (`?table`) - object after which objects should be selected
   * `batch_size` (`?number`) - number of tuples to process per one request to storage
   * `timeout` (`?number`) - `vshard.call` timeout (in seconds)
@@ -261,7 +263,8 @@ crud.select('customers', {{'<=', 'age', 35}})
 ### Pairs
 
 You can iterate across a distributed space using the `crud.pairs` function.
-Its arguments are the same as [`crud.select`](#select) arguments.
+Its arguments are the same as [`crud.select`](#select) arguments,
+but negative `first` values aren't allowed.
 
 **Example:**
 

crud/common/utils.lua

@@ -173,4 +173,26 @@ function utils.unflatten_rows(rows, metadata)
     return result
 end
 
+local inverted_tarantool_iters = {
+    [box.index.EQ] = box.index.REQ,
+    [box.index.GT] = box.index.LT,
+    [box.index.GE] = box.index.LE,
+    [box.index.LT] = box.index.GT,
+    [box.index.LE] = box.index.GE,
+    [box.index.REQ] = box.index.EQ,
+}
+
+function utils.invert_tarantool_iter(iter)
+    local inverted_iter = inverted_tarantool_iters[iter]
+    assert(inverted_iter ~= nil, "Unsupported Tarantool iterator: " .. tostring(iter))
+    return inverted_iter
+end
+
+function utils.reverse_inplace(t)
+    for i = 1,#t - 1 do
+        t[i], t[#t - i + 1] = t[#t - i + 1], t[i]
+    end
+    return t
+end
+
 return utils

crud/select.lua

@@ -119,7 +119,7 @@ end
 local function build_select_iterator(space_name, user_conditions, opts)
     dev_checks('string', '?table', {
         after = '?table',
-        limit = '?number',
+        first = '?number',
         timeout = '?number',
         batch_size = '?number',
     })
@@ -132,10 +132,6 @@ local function build_select_iterator(space_name, user_conditions, opts)
 
     local batch_size = opts.batch_size or DEFAULT_BATCH_SIZE
 
-    if opts.limit ~= nil and opts.limit < 0 then
-        return nil, SelectError:new("limit should be >= 0")
-    end
-
     -- check conditions
     local conditions, err = select_conditions.parse(user_conditions)
     if err ~= nil then
@@ -153,25 +149,26 @@ local function build_select_iterator(space_name, user_conditions, opts)
     end
     local space_format = space:format()
 
+    -- set after tuple
+    local after_tuple = utils.flatten(opts.after, space_format)
+
     -- plan select
     local plan, err = select_plan.new(space, conditions, {
-        limit = opts.limit,
+        first = opts.first,
+        after_tuple = after_tuple,
     })
 
     if err ~= nil then
         return nil, SelectError:new("Failed to plan select: %s", err)
     end
 
-    -- set limit and replicasets to select from
+    -- set replicasets to select from
     local replicasets_to_select = replicasets
 
     if plan.sharding_key ~= nil then
         replicasets_to_select = get_replicasets_by_sharding_key(plan.sharding_key)
     end
 
-    -- set after tuple
-    local after_tuple = utils.flatten(opts.after, space_format)
-
     -- generate tuples comparator
     local scan_index = space.index[plan.index_id]
     local primary_index = space.index[0]
@@ -191,7 +188,6 @@ local function build_select_iterator(space_name, user_conditions, opts)
         comparator = tuples_comparator,
 
         plan = plan,
-        after_tuple = after_tuple,
 
         batch_size = batch_size,
         replicasets = replicasets_to_select,
@@ -205,16 +201,20 @@ end
 function select_module.pairs(space_name, user_conditions, opts)
     checks('string', '?table', {
         after = '?table',
-        limit = '?number',
+        first = '?number',
         timeout = '?number',
         batch_size = '?number',
     })
 
     opts = opts or {}
 
+    if opts.first ~= nil and opts.first < 0 then
+        error(string.format("Negative first isn't allowed for pairs"))
+    end
+
     local iter, err = build_select_iterator(space_name, user_conditions, {
         after = opts.after,
-        limit = opts.limit,
+        first = opts.first,
         timeout = opts.timeout,
         batch_size = opts.batch_size,
     })
@@ -247,16 +247,22 @@ end
 function select_module.call(space_name, user_conditions, opts)
     checks('string', '?table', {
         after = '?table',
-        limit = '?number',
+        first = '?number',
         timeout = '?number',
         batch_size = '?number',
     })
 
     opts = opts or {}
 
+    if opts.first ~= nil and opts.first < 0 then
+        if opts.after == nil then
+            return nil, SelectError:new("Negative first should be specified only with after option")
+        end
+    end
+
     local iter, err = build_select_iterator(space_name, user_conditions, {
         after = opts.after,
-        limit = opts.limit,
+        first = opts.first,
         timeout = opts.timeout,
         batch_size = opts.batch_size,
     })
@@ -268,16 +274,20 @@ function select_module.call(space_name, user_conditions, opts)
     local tuples = {}
 
     while iter:has_next() do
-        local obj, err = iter:get()
+        local tuple, err = iter:get()
         if err ~= nil then
             return nil, SelectError:new("Failed to get next object: %s", err)
         end
 
-        if obj == nil then
+        if tuple == nil then
             break
         end
 
-        table.insert(tuples, obj)
+        table.insert(tuples, tuple)
+    end
+
+    if opts.first ~= nil and opts.first < 0 then
+        utils.reverse_inplace(tuples)
     end
 
     return {

crud/select/comparators.lua

@@ -177,7 +177,7 @@ local array_cmp_funcs_by_operators = {
 --]=]
 function comparators.get_cmp_operator(tarantool_iter)
     local cmp_operator = cmp_operators_by_tarantool_iter[tarantool_iter]
-    assert(cmp_operator ~= nil, 'Unsupported Tarantool iterator %q', tarantool_iter)
+    assert(cmp_operator ~= nil, 'Unsupported Tarantool iterator: ' .. tostring(tarantool_iter))
 
     return cmp_operator
 end

crud/select/iterator.lua

@@ -20,7 +20,6 @@ function Iterator.new(opts)
         iteration_func = 'function',
 
         plan = 'table',
-        after_tuple = '?table',
 
         batch_size = 'number',
         replicasets = 'table',
@@ -34,7 +33,7 @@ function Iterator.new(opts)
         iteration_func = opts.iteration_func,
 
         plan = opts.plan,
-        after_tuple = opts.after_tuple,
+
         timeout = opts.timeout,
 
         replicasets = table.copy(opts.replicasets),
@@ -56,7 +55,7 @@ function Iterator.new(opts)
 
     setmetatable(iter, Iterator)
 
-    iter:_update_replicasets_tuples(iter.after_tuple)
+    iter:_update_replicasets_tuples(iter.plan.after_tuple)
 
     return iter
 end

crud/select/plan.lua

@@ -1,6 +1,7 @@
 local errors = require('errors')
 
 local select_conditions = require('crud.select.conditions')
+local utils = require('crud.common.utils')
 local dev_checks = require('crud.common.dev_checks')
 
 local select_plan = {}
@@ -86,8 +87,10 @@ end
 
 function select_plan.new(space, conditions, opts)
     dev_checks('table', '?table', {
-        limit = '?number',
+        first = '?number',
+        after_tuple = '?table',
     })
+
     conditions = conditions ~= nil and conditions or {}
     opts = opts or {}
 
@@ -135,8 +138,26 @@ function select_plan.new(space, conditions, opts)
         scan_value = {}
     end
 
-    -- set total_tuples_count
-    local total_tuples_count = opts.limit
+    -- handle opts.first
+    local total_tuples_count
+    local scan_after_tuple = opts.after_tuple
+
+    if opts.first ~= nil then
+        total_tuples_count = math.abs(opts.first)
+
+        if opts.first < 0 then
+            scan_iter = utils.invert_tarantool_iter(scan_iter)
+
+            -- scan condition becomes border consition
+            scan_condition_num = nil
+
+            if scan_after_tuple ~= nil then
+                scan_value = utils.extract_key(scan_after_tuple, scan_index.parts)
+            else
+                scan_value = nil
+            end
+        end
+    end
 
     local sharding_index = primary_index -- XXX: only sharding by primary key is supported
 
@@ -156,6 +177,7 @@ function select_plan.new(space, conditions, opts)
         space_name = space_name,
         index_id = scan_index.id,
         scan_value = scan_value,
+        after_tuple = scan_after_tuple,
         scan_condition_num = scan_condition_num,
         iter = scan_iter,
         total_tuples_count = total_tuples_count,

test/integration/pairs_test.lua

@@ -296,3 +296,28 @@ add('test_le_condition_with_index', function(g)
     t.assert_equals(err, nil)
     t.assert_equals(objects, get_by_ids(customers, {1})) -- in age order
 end)
+
+add('test_negative_first', function(g)
+    local customers = insert_customers(g,{
+        {
+            id = 1, name = "Elizabeth", last_name = "Jackson",
+            age = 12, city = "New York",
+        }, {
+            id = 2, name = "Mary", last_name = "Brown",
+            age = 46, city = "Los Angeles",
+        }, {
+            id = 3, name = "David", last_name = "Smith",
+            age = 33, city = "Los Angeles",
+        },
+    })
+
+    table.sort(customers, function(obj1, obj2) return obj1.id < obj2.id end)
+
+    -- negative first
+    t.assert_error_msg_contains("Negative first isn't allowed for pairs", function()
+        g.cluster.main_server.net_box:eval([[
+            local crud = require('crud')
+            crud.pairs('customers', nil, {first = -10})
+        ]])
+    end)
+end)